@devries-cameron20 / t3code-1 / commits / 36dac688a1
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): restart continuations retry failed starts and respect a parent's stop (#27222)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Julius Marminge committed Oct 29, 2026 at 18:12 UTC 36dac688a17c6de1c6398e62d01546fb787c185b
6 files changed +464 -6
apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts
+59
@@ -2448,6 +2448,65 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => {
2448 const task = settled.subagents.find((row) => row.id === child.taskId);
2449 assert.equal(task?.status, "cancelled");
2450 assert.equal(task?.completionDelivery?.state, "claimed");
2451 +
2452 + // The parent's Stop committed, but the restart cut the effect that stops its children.
2453 + const stoppedChild = yield* seedRestartCancelledChild({
2454 + parentThreadId: threadId,
2455 + projectId,
2456 + parentRunId: runId,
2457 + rootNodeId,
2458 + name: "restart-parent-stopped-child",
2459 + completionWake: "always",
2460 + continuationPending: true,
2461 + now,
2462 + });
2463 + yield* (yield* EventSink.EventSinkV2).write({
2464 + commandId: CommandId.make("command:restart-declined-parent:stop"),
2465 + events: [
2466 + {
2467 + id: EventId.make("event:restart-declined-parent:interrupt-request"),
2468 + type: "turn-item.updated",
2469 + threadId,
2470 + runId,
2471 + nodeId: rootNodeId,
2472 + providerInstanceId: modelSelection.instanceId,
2473 + occurredAt: now,
2474 + payload: {
2475 + id: TurnItemId.make("turn-item:restart-declined-parent:interrupt-request"),
2476 + type: "run_interrupt_request",
2477 + threadId,
2478 + runId,
2479 + nodeId: rootNodeId,
2480 + providerThreadId: null,
2481 + providerTurnId: null,
2482 + nativeItemRef: null,
2483 + parentItemId: null,
2484 + ordinal: 90,
2485 + status: "completed",
2486 + title: "Interrupt requested",
2487 + startedAt: now,
2488 + completedAt: now,
2489 + updatedAt: now,
2490 + message: "Stopped by user",
2491 + },
2492 + },
2493 + ],
2494 + });
2495 + yield* continueRestartedRun({
2496 + threadId: stoppedChild.childThreadId,
2497 + sourceRunId: stoppedChild.childRunId,
2498 + }).pipe(Effect.provide(ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true })));
2499 + const afterStop = yield* orchestrator.getThreadProjection(threadId);
2500 + assert.equal(
2501 + afterStop.subagents.find((row) => row.id === stoppedChild.taskId)?.status,
2502 + "cancelled",
2503 + );
2504 + assert.deepEqual(
2505 + (yield* orchestrator.getThreadProjection(stoppedChild.childThreadId)).runs.map(
2506 + (run) => run.id,
2507 + ),
2508 + [stoppedChild.childRunId],
2509 + );
2510 }),
2511 );
2512
apps/server/src/orchestration-v2/Orchestrator.ts
+49 -2
@@ -107,6 +107,8 @@ import { ContextHandoffServiceV2 } from "./ContextHandoffService.ts";
107 import { notificationTurnItem } from "@t3tools/provider-core/server/notification";
108 import {
109 cancelledBeforeStart,
110 + continuationFailedBeforePrompt,
111 + continuationStartRetryDelayMs,
112 isRestartContinuationSource,
113 runOutcome,
114 } from "./RestartBackgroundNote.ts";
@@ -2209,6 +2211,35 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
2211 ),
2212 );
2213
2214 + // Under load a resumed provider can time out before taking the prompt; the restart still owes the turn.
2215 + const retryContinuationStart = Effect.fn("Orchestrator.retryContinuationStart")(function* (
2216 + threadId: ThreadId,
2217 + runId: RunId,
2218 + ) {
2219 + const projection = yield* projectionStore.getThreadRecords(
2220 + threadId,
2221 + ["runs", "providerTurns", "turnItems"],
2222 + { turnItemTypes: ["error"], turnItemRunIds: [runId] },
2223 + );
2224 + const run = projection.runs.find((candidate) => candidate.id === runId);
2225 + if (run === undefined) return;
2226 + const delayMs = continuationStartRetryDelayMs(run, projection);
2227 + if (delayMs === undefined) return;
2228 + const commandId = CommandId.make(`command:restart-continuation-retry:${run.id}`);
2229 + yield* writeSystemEvents(
2230 + [],
2231 + [
2232 + {
2233 + id: `effect:restart-continuation:${run.id}`,
2234 + commandId,
2235 + threadId,
2236 + request: { type: "provider-runtime.continue", sourceRunId: run.id },
2237 + availableAt: DateTime.add(yield* DateTime.now, { milliseconds: delayMs }),
2238 + },
2239 + ],
2240 + );
2241 + });
2242 +
2243 // A hold waits for a person to review what they queued. Nobody reviews messages from agents,
2244 // the server or scheduled tasks, so a later completed run releases those unless it was stopped.
2245 const releaseUnattendedHolds = Effect.fn("Orchestrator.releaseUnattendedHolds")(function* (
@@ -5188,7 +5219,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
5219 const source = projection.runs.find((run) => run.id === command.restartContinuationOfRunId);
5220 if (
5221 !source ||
5191 - !isRestartContinuationSource(source, projection.providerTurns) ||
5222 + !(
5223 + isRestartContinuationSource(source, projection.providerTurns) ||
5224 + continuationFailedBeforePrompt(source, projection.providerTurns)
5225 + ) ||
5226 projection.thread.archivedAt !== null ||
5227 projection.thread.deletedAt !== null ||
5228 projection.thread.providerInstanceId !== source.providerInstanceId ||
@@ -10419,7 +10453,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
10453 // A resumed queue can have a lower ordinal than a pending continuation source.
10454 // Stop must reach those sources even when interrupting the resumed run succeeds.
10455 if (
10422 - run.status === "cancelled" &&
10456 + (run.status === "cancelled" ||
10457 + (run.status === "failed" && run.restartContinuationOfRunId !== undefined)) &&
10458 (yield* awaitsRestartContinuation(run).pipe(mapDispatchError(input.command)))
10459 ) {
10460 yield* markStoppedRun({ ...input, thread, run });
@@ -13129,6 +13164,18 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
13164 // while writing child events, so nesting parent inside child would invert the order.
13165 // The keyed semaphores are neither reentrant nor deadlock-aware.
13166 if (delegatedTaskTerminalStatus(run.status) !== null) {
13167 + // Before the parent hears of the failure: a pending continuation keeps the task open.
13168 + if (run.status === "failed" && run.restartContinuationOfRunId !== undefined) {
13169 + yield* threadDispatch.withLock(threadId, retryContinuationStart(threadId, run.id)).pipe(
13170 + Effect.catchCause((cause) =>
13171 + Effect.logWarning("Failed to schedule a restart continuation retry", {
13172 + threadId,
13173 + runId: run.id,
13174 + cause,
13175 + }),
13176 + ),
13177 + );
13178 + }
13179 yield* threadDispatch.withLock(
13180 threadId,
13181 finalizeDelegatedCompletionDelivery(threadId, run.id),
apps/server/src/orchestration-v2/RestartBackgroundNote.ts
+45 -1
@@ -6,7 +6,7 @@ import type {
6 OrchestrationV2RunAttempt,
7 OrchestrationV2TurnItem,
8 } from "@t3tools/contracts";
9 -import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
9 +import { latestRootProviderFailure, runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
10
11 type Work = OrchestrationV2RestartCancelledBackgroundWork;
12 type Attempt = Pick<OrchestrationV2RunAttempt, "id" | "runId">;
@@ -181,6 +181,50 @@ export function cancelledBeforeStart(
181 return run.status === "cancelled" && neverStarted(run);
182 }
183
184 +/** The provider never accepted this continuation's prompt, so starting it again repeats nothing. */
185 +export function continuationFailedBeforePrompt(
186 + run: Pick<OrchestrationV2Run, "status" | "activeAttemptId" | "restartContinuationOfRunId">,
187 + providerTurns: ReadonlyArray<Pick<ProviderTurnState, "runAttemptId">>,
188 +): boolean {
189 + return (
190 + run.status === "failed" &&
191 + run.restartContinuationOfRunId !== undefined &&
192 + !providerTurns.some((turn) => turn.runAttemptId === run.activeAttemptId)
193 + );
194 +}
195 +
196 +const CONTINUATION_START_RETRY_DELAYS_MS = [60_000, 180_000];
197 +
198 +/** Backoff before restarting a continuation that failed to start, or undefined to let it fail. */
199 +export function continuationStartRetryDelayMs(
200 + run: OrchestrationV2Run,
201 + projection: {
202 + readonly runs: ReadonlyArray<OrchestrationV2Run>;
203 + readonly providerTurns: ReadonlyArray<Pick<ProviderTurnState, "runAttemptId">>;
204 + readonly turnItems: Parameters<typeof latestRootProviderFailure>[1];
205 + },
206 +): number | undefined {
207 + if (!continuationFailedBeforePrompt(run, projection.providerTurns)) return;
208 + // These fail the same way however often they start.
209 + const failureClass = latestRootProviderFailure(run, projection.turnItems)?.class;
210 + if (
211 + failureClass === "usage_limit" ||
212 + failureClass === "permission_error" ||
213 + failureClass === "validation_error"
214 + )
215 + return;
216 + let failedStarts = 0;
217 + for (
218 + let current: OrchestrationV2Run | undefined = run;
219 + current !== undefined && continuationFailedBeforePrompt(current, projection.providerTurns);
220 + current = projection.runs.find(
221 + (candidate) => candidate.id === current?.restartContinuationOfRunId,
222 + )
223 + )
224 + failedStarts += 1;
225 + return CONTINUATION_START_RETRY_DELAYS_MS[failedStarts - 1];
226 +}
227 +
228 export function isRestartContinuationSource(
229 source: Pick<OrchestrationV2Run, "status" | "activeAttemptId" | "restartCancelledBackgroundWork">,
230 providerTurns: ReadonlyArray<Pick<ProviderTurnState, "runAttemptId" | "status">>,
apps/server/src/orchestration-v2/RestartContinuation.test.ts
+120 -2
@@ -20,6 +20,7 @@ import * as Effect from "effect/Effect";
20 import * as Layer from "effect/Layer";
21 import * as ServerSettings from "../serverSettings.ts";
22 import { restartContinuationRun, continueRestartedRun } from "./RestartContinuation.ts";
23 +import { continuationStartRetryDelayMs } from "./RestartBackgroundNote.ts";
24 import * as ThreadManagementService from "./ThreadManagementService.ts";
25 import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts";
26 import * as ProjectionStore from "./ProjectionStore.ts";
@@ -46,6 +47,8 @@ function makeProjection() {
47 providerInstanceId: instanceId,
48 archivedAt: null,
49 deletedAt: null,
50 + lineage: { parentThreadId: null, relationshipToParent: null },
51 + forkedFrom: null,
52 },
53 runs: [
54 {
@@ -839,10 +842,13 @@ it.effect("does not cancel or resume a run that completes while shutdown intent
842 }),
843 );
844
842 -const continuationDispatches = (projection: OrchestrationV2ThreadProjection) =>
845 +const continuationDispatches = (
846 + projection: OrchestrationV2ThreadProjection,
847 + sourceRunId: RunId = runId,
848 +) =>
849 Effect.gen(function* () {
850 const dispatches: Array<Extract<OrchestrationV2Command, { type: "message.dispatch" }>> = [];
845 - yield* continueRestartedRun({ threadId, sourceRunId: runId }).pipe(
851 + yield* continueRestartedRun({ threadId, sourceRunId }).pipe(
852 Effect.provide(
853 Layer.merge(
854 Layer.mock(ThreadManagementService.ThreadManagementService)({
@@ -943,6 +949,118 @@ it.effect("continues a resumed queued run that ran after an earlier continuation
949 }),
950 );
951
952 +it("backs off twice from a continuation the provider never took, then lets it fail", () => {
953 + const base = makeProjection();
954 + const failedStart = (id: string, of: RunId) => ({
955 + ...base.runs[0]!,
956 + id: RunId.make(id),
957 + activeAttemptId: RunAttemptId.make(`attempt:${id}`),
958 + status: "failed" as const,
959 + restartContinuationOfRunId: of,
960 + });
961 + const first = failedStart("run:retry-1", runId);
962 + const second = failedStart("run:retry-2", first.id);
963 + const third = failedStart("run:retry-3", second.id);
964 + const projection = (failureClass = "provider_error", providerTurns = base.providerTurns) =>
965 + ({
966 + runs: [{ ...base.runs[0]!, status: "cancelled" }, first, second, third],
967 + providerTurns,
968 + turnItems: [first, second, third].map((run) => ({
969 + id: `error:${run.id}`,
970 + type: "error",
971 + status: "failed",
972 + runId: run.id,
973 + nodeId: run.rootNodeId,
974 + ordinal: 1,
975 + updatedAt: now,
976 + failure: { class: failureClass, message: "Failed.", code: null, retryable: null },
977 + })),
978 + }) as unknown as Parameters<typeof continuationStartRetryDelayMs>[1];
979 + assert.deepEqual(
980 + [first, second, third].map((run) => continuationStartRetryDelayMs(run, projection())),
981 + [60_000, 180_000, undefined],
982 + );
983 + for (const failureClass of ["usage_limit", "permission_error", "validation_error"])
984 + assert.isUndefined(continuationStartRetryDelayMs(first, projection(failureClass)));
985 + // Once the provider took the prompt, starting again would repeat the turn.
986 + assert.isUndefined(
987 + continuationStartRetryDelayMs(
988 + first,
989 + projection("provider_error", [
990 + { ...base.providerTurns[0]!, runAttemptId: first.activeAttemptId, status: "failed" },
991 + ] as never),
992 + ),
993 + );
994 +});
995 +
996 +it.effect("continues again after a continuation fails before its prompt", () =>
997 + Effect.gen(function* () {
998 + const cut = cutMidTurn();
999 + const failedStart = {
1000 + ...cut.runs[0]!,
1001 + id: RunId.make("run:failed-start"),
1002 + ordinal: 2,
1003 + activeAttemptId: RunAttemptId.make("attempt:failed-start"),
1004 + status: "failed",
1005 + restartContinuationOfRunId: runId,
1006 + };
1007 + const dispatches = yield* continuationDispatches(
1008 + { ...cut, runs: [...cut.runs, failedStart] } as unknown as OrchestrationV2ThreadProjection,
1009 + failedStart.id,
1010 + );
1011 + assert.deepEqual(
1012 + dispatches.map((command) => command.restartContinuationOfRunId),
1013 + [failedStart.id],
1014 + );
1015 + }),
1016 +);
1017 +
1018 +it.effect.each([
1019 + ["running", null, 0],
1020 + ["completed", "Done.", 1],
1021 +] as const)(
1022 + "a parent's Stop on the task's run blocks the child only while the task is %s",
1023 + ([status, result, expected]) =>
1024 + Effect.gen(function* () {
1025 + const parentThreadId = ThreadId.make("thread:parent");
1026 + const parentRunId = RunId.make("run:parent");
1027 + const taskId = NodeId.make("node:task");
1028 + const cut = cutMidTurn();
1029 + const child = {
1030 + ...cut,
1031 + thread: {
1032 + ...cut.thread,
1033 + lineage: { parentThreadId, relationshipToParent: "subagent" },
1034 + forkedFrom: { type: "node", nodeId: taskId },
1035 + },
1036 + } as unknown as OrchestrationV2ThreadProjection;
1037 + const parent = {
1038 + thread: { id: parentThreadId },
1039 + subagents: [{ id: taskId, runId: parentRunId, origin: "app_owned", status, result }],
1040 + turnItems: [{ id: "turn-item:stop", runId: parentRunId, type: "run_interrupt_request" }],
1041 + };
1042 + let dispatched = 0;
1043 + yield* continueRestartedRun({ threadId, sourceRunId: runId }).pipe(
1044 + Effect.provide(
1045 + Layer.merge(
1046 + Layer.mock(ThreadManagementService.ThreadManagementService)({
1047 + getThreadRecords: (id) =>
1048 + Effect.succeed((id === parentThreadId ? parent : child) as never),
1049 + recoverDelegatedTask: () => Effect.void,
1050 + dispatch: () =>
1051 + Effect.sync(() => {
1052 + dispatched += 1;
1053 + return {} as never;
1054 + }),
1055 + }),
1056 + ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
1057 + ),
1058 + ),
1059 + );
1060 + assert.equal(dispatched, expected);
1061 + }),
1062 +);
1063 +
1064 it.effect("does not continue a run the user asked to stop before the restart", () =>
1065 Effect.gen(function* () {
1066 const texts = yield* continuationTexts({
apps/server/src/orchestration-v2/RestartContinuation.ts
+32 -1
@@ -2,6 +2,7 @@ import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
2 import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
3 import {
4 CommandId,
5 + isOrchestrationV2WorkActive,
6 latestProviderTurnForAttempt,
7 MessageId,
8 type OrchestrationV2Run,
@@ -19,6 +20,7 @@ import {
20 awaitingProviderThreadId,
21 cancelledRosterTaskWork,
22 cancelledTurnItemWork,
23 + continuationFailedBeforePrompt,
24 isAwaitedWork,
25 isRestartContinuationSource,
26 cancelledBeforeStart,
@@ -200,7 +202,10 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta
202 const source = projection.runs.find((run) => run.id === input.sourceRunId);
203 if (
204 !source ||
203 - !isRestartContinuationSource(source, projection.providerTurns) ||
205 + !(
206 + isRestartContinuationSource(source, projection.providerTurns) ||
207 + continuationFailedBeforePrompt(source, projection.providerTurns)
208 + ) ||
209 restartContinuationBlocked(
210 projection,
211 source,
@@ -255,6 +260,32 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta
260 )
261 )
262 return;
263 + // A parent's Stop reaches its open delegated tasks through an effect a restart can cut short.
264 + const { lineage, forkedFrom } = projection.thread;
265 + const taskId =
266 + source.delegatedTaskId ?? (forkedFrom?.type === "node" ? forkedFrom.nodeId : undefined);
267 + if (
268 + lineage.relationshipToParent === "subagent" &&
269 + lineage.parentThreadId !== null &&
270 + taskId !== undefined
271 + ) {
272 + const task = (yield* threads.getThreadRecords(lineage.parentThreadId, ["subagents"], {
273 + subagentIds: [taskId],
274 + })).subagents.find((candidate) => candidate.id === taskId);
275 + // A settled task no longer owns later work in its thread.
276 + const parentRunId =
277 + task !== undefined && task.result === null && isOrchestrationV2WorkActive(task.status)
278 + ? task.runId
279 + : null;
280 + if (
281 + parentRunId !== null &&
282 + (yield* threads.getThreadRecords(lineage.parentThreadId, ["turnItems"], {
283 + turnItemTypes: ["run_interrupt_request"],
284 + turnItemRunIds: [parentRunId],
285 + })).turnItems.length > 0
286 + )
287 + return;
288 + }
289 const sourceMessage = sourceRecords.messages.find(
290 (message) => message.id === source.userMessageId,
291 );
apps/server/src/orchestration-v2/runtimeLayer.test.ts
+159
@@ -5465,6 +5465,165 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
5465 }),
5466 );
5467
5468 + it.effect.each([false, true])(
5469 + "restarts a continuation that failed before its prompt, unless stopped (%s)",
5470 + (stopped) =>
5471 + Effect.gen(function* () {
5472 + const orchestrator = yield* Orchestrator.OrchestratorV2;
5473 + const eventSink = yield* EventSink.EventSinkV2;
5474 + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
5475 + const threadId = ThreadId.make(`runtime-layer-continuation-start-retry-${stopped}`);
5476 + const messageId = (key: string) => MessageId.make(`${threadId}:${key}`);
5477 + yield* orchestrator.dispatch({
5478 + type: "thread.create",
5479 + createdBy: "user",
5480 + creationSource: "web",
5481 + commandId: CommandId.make(`${threadId}:create`),
5482 + threadId,
5483 + projectId: ProjectId.make(`${threadId}:project`),
5484 + title: "Continuation start retry",
5485 + modelSelection,
5486 + runtimeMode: "full-access",
5487 + interactionMode: "default",
5488 + branch: null,
5489 + worktreePath: process.cwd(),
5490 + });
5491 + const runFor = (key: string) =>
5492 + Effect.map(orchestrator.getThreadProjection(threadId), ({ runs }) =>
5493 + runs.find((candidate) => candidate.userMessageId === messageId(key)),
5494 + );
5495 + const write = (
5496 + key: string,
5497 + payload: OrchestrationV2Run,
5498 + extra: Parameters<typeof eventSink.write>[0]["events"] = [],
5499 + ) =>
5500 + Effect.gen(function* () {
5501 + const now = yield* DateTime.now;
5502 + yield* eventSink.write({
5503 + events: [
5504 + ...extra,
5505 + {
5506 + id: EventId.make(`${threadId}:${key}`),
5507 + type: "run.updated",
5508 + threadId,
5509 + runId: payload.id,
5510 + providerInstanceId: payload.providerInstanceId,
5511 + occurredAt: now,
5512 + payload: { ...payload, startedAt: now, completedAt: now },
5513 + },
5514 + ],
5515 + });
5516 + });
5517 + yield* orchestrator.dispatch({
5518 + type: "message.dispatch",
5519 + createdBy: "user",
5520 + creationSource: "web",
5521 + commandId: CommandId.make(`${threadId}:message:cut`),
5522 + threadId,
5523 + messageId: messageId("cut"),
5524 + text: "Work that the restart cut",
5525 + attachments: [],
5526 + modelSelection,
5527 + dispatchMode: { type: "start_immediately" },
5528 + });
5529 + const cut = (yield* runFor("cut"))!;
5530 + yield* write("cut:cancelled", { ...cut, status: "cancelled" });
5531 + yield* orchestrator.dispatch({
5532 + type: "message.dispatch",
5533 + createdBy: "agent",
5534 + creationSource: "server",
5535 + commandId: CommandId.make(`${threadId}:message:continuation`),
5536 + threadId,
5537 + messageId: messageId("continuation"),
5538 + text: "Continue where you left off.",
5539 + attachments: [],
5540 + modelSelection,
5541 + dispatchMode: { type: "start_immediately" },
5542 + restartContinuationOfRunId: cut.id,
5543 + });
5544 + yield* orchestrator.dispatch({
5545 + type: "message.dispatch",
5546 + createdBy: "user",
5547 + creationSource: "web",
5548 + commandId: CommandId.make(`${threadId}:message:queued`),
5549 + threadId,
5550 + messageId: messageId("queued"),
5551 + text: "Queued behind the continuation",
5552 + attachments: [],
5553 + modelSelection,
5554 + dispatchMode: { type: "queue_after_active" },
5555 + });
5556 + const continuation = (yield* runFor("continuation"))!;
5557 + assert.isNotNull(continuation.rootNodeId);
5558 + const queued = (yield* runFor("queued"))!;
5559 + // The failure holds the queue after the reactor scheduled any retry.
5560 + const held = yield* eventSink
5561 + .stream({ threadId, afterSequence: yield* orchestrator.getThreadEventSequence(threadId) })
5562 + .pipe(
5563 + Stream.filter(
5564 + (stored) =>
5565 + stored.event.type === "run.updated" &&
5566 + stored.event.payload.id === queued.id &&
5567 + stored.event.payload.queueHeld === true,
5568 + ),
5569 + Stream.runHead,
5570 + Effect.forkScoped,
5571 + );
5572 + yield* Effect.yieldNow;
5573 + const now = yield* DateTime.now;
5574 + yield* write("continuation:failed", { ...continuation, status: "failed" }, [
5575 + {
5576 + id: EventId.make(`${threadId}:continuation:error`),
5577 + type: "turn-item.updated",
5578 + threadId,
5579 + runId: continuation.id,
5580 + nodeId: continuation.rootNodeId,
5581 + providerInstanceId: continuation.providerInstanceId,
5582 + occurredAt: now,
5583 + payload: {
5584 + id: TurnItemId.make(`${threadId}:continuation:error`),
5585 + type: "error",
5586 + threadId,
5587 + runId: continuation.id,
5588 + nodeId: continuation.rootNodeId,
5589 + providerThreadId: continuation.providerThreadId,
5590 + providerTurnId: null,
5591 + nativeItemRef: null,
5592 + parentItemId: null,
5593 + ordinal: 2,
5594 + status: "failed",
5595 + title: "Provider failure",
5596 + startedAt: now,
5597 + completedAt: now,
5598 + updatedAt: now,
5599 + failure: {
5600 + class: "provider_error",
5601 + message: "Claude timed out while registering its MCP tools.",
5602 + code: null,
5603 + retryable: null,
5604 + },
5605 + },
5606 + },
5607 + ]);
5608 + yield* Fiber.join(held);
5609 + const retriesOf = Effect.map(orchestrator.getThreadProjection(threadId), ({ runs }) =>
5610 + runs.filter((run) => run.restartContinuationOfRunId === continuation.id),
5611 + );
5612 + yield* worker.drain();
5613 + assert.deepEqual(yield* retriesOf, [], "the retry waits out its backoff");
5614 + if (stopped) {
5615 + yield* orchestrator.dispatch({
5616 + type: "thread.stop",
5617 + commandId: CommandId.make(`${threadId}:stop`),
5618 + threadId,
5619 + });
5620 + }
5621 + yield* TestClock.adjust("61 seconds");
5622 + yield* worker.drain();
5623 + assert.lengthOf(yield* retriesOf, stopped ? 0 : 1);
5624 + }),
5625 + );
5626 +
5627 it.effect.each(["startup", "shutdown"] as const)(
5628 "preserves and holds queued messages across %s until explicitly resumed",
5629 (trigger) =>