The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 12:00 UTC
fix(server): restart continuations retry failed starts and respect a parent's stop (#27222)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Julius Marminge committed
Oct 29, 2026 at 18:12 UTC
36dac688a17c6de1c6398e62d01546fb787c185b
6 files changed
+464
-6
apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts
+59
@@ -2448,6 +2448,65 @@ it.layer(layerTest)("delegated tasks across a server restart", (it) => {
2448
const task = settled.subagents.find((row) => row.id === child.taskId);
2449
assert.equal(task?.status, "cancelled");
2450
assert.equal(task?.completionDelivery?.state, "claimed");
2451
+
2452
+ // The parent's Stop committed, but the restart cut the effect that stops its children.
2453
+ const stoppedChild = yield* seedRestartCancelledChild({
2454
+ parentThreadId: threadId,
2455
+ projectId,
2456
+ parentRunId: runId,
2457
+ rootNodeId,
2458
+ name: "restart-parent-stopped-child",
2459
+ completionWake: "always",
2460
+ continuationPending: true,
2461
+ now,
2462
+ });
2463
+ yield* (yield* EventSink.EventSinkV2).write({
2464
+ commandId: CommandId.make("command:restart-declined-parent:stop"),
2465
+ events: [
2466
+ {
2467
+ id: EventId.make("event:restart-declined-parent:interrupt-request"),
2468
+ type: "turn-item.updated",
2469
+ threadId,
2470
+ runId,
2471
+ nodeId: rootNodeId,
2472
+ providerInstanceId: modelSelection.instanceId,
2473
+ occurredAt: now,
2474
+ payload: {
2475
+ id: TurnItemId.make("turn-item:restart-declined-parent:interrupt-request"),
2476
+ type: "run_interrupt_request",
2477
+ threadId,
2478
+ runId,
2479
+ nodeId: rootNodeId,
2480
+ providerThreadId: null,
2481
+ providerTurnId: null,
2482
+ nativeItemRef: null,
2483
+ parentItemId: null,
2484
+ ordinal: 90,
2485
+ status: "completed",
2486
+ title: "Interrupt requested",
2487
+ startedAt: now,
2488
+ completedAt: now,
2489
+ updatedAt: now,
2490
+ message: "Stopped by user",
2491
+ },
2492
+ },
2493
+ ],
2494
+ });
2495
+ yield* continueRestartedRun({
2496
+ threadId: stoppedChild.childThreadId,
2497
+ sourceRunId: stoppedChild.childRunId,
2498
+ }).pipe(Effect.provide(ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true })));
2499
+ const afterStop = yield* orchestrator.getThreadProjection(threadId);
2500
+ assert.equal(
2501
+ afterStop.subagents.find((row) => row.id === stoppedChild.taskId)?.status,
2502
+ "cancelled",
2503
+ );
2504
+ assert.deepEqual(
2505
+ (yield* orchestrator.getThreadProjection(stoppedChild.childThreadId)).runs.map(
2506
+ (run) => run.id,
2507
+ ),
2508
+ [stoppedChild.childRunId],
2509
+ );
2510
}),
2511
);
2512
apps/server/src/orchestration-v2/Orchestrator.ts
+49
-2
@@ -107,6 +107,8 @@ import { ContextHandoffServiceV2 } from "./ContextHandoffService.ts";
107
import { notificationTurnItem } from "@t3tools/provider-core/server/notification";
108
import {
109
cancelledBeforeStart,
110
+ continuationFailedBeforePrompt,
111
+ continuationStartRetryDelayMs,
112
isRestartContinuationSource,
113
runOutcome,
114
} from "./RestartBackgroundNote.ts";
@@ -2209,6 +2211,35 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
2211
),
2212
);
2213
2214
+ // Under load a resumed provider can time out before taking the prompt; the restart still owes the turn.
2215
+ const retryContinuationStart = Effect.fn("Orchestrator.retryContinuationStart")(function* (
2216
+ threadId: ThreadId,
2217
+ runId: RunId,
2218
+ ) {
2219
+ const projection = yield* projectionStore.getThreadRecords(
2220
+ threadId,
2221
+ ["runs", "providerTurns", "turnItems"],
2222
+ { turnItemTypes: ["error"], turnItemRunIds: [runId] },
2223
+ );
2224
+ const run = projection.runs.find((candidate) => candidate.id === runId);
2225
+ if (run === undefined) return;
2226
+ const delayMs = continuationStartRetryDelayMs(run, projection);
2227
+ if (delayMs === undefined) return;
2228
+ const commandId = CommandId.make(`command:restart-continuation-retry:${run.id}`);
2229
+ yield* writeSystemEvents(
2230
+ [],
2231
+ [
2232
+ {
2233
+ id: `effect:restart-continuation:${run.id}`,
2234
+ commandId,
2235
+ threadId,
2236
+ request: { type: "provider-runtime.continue", sourceRunId: run.id },
2237
+ availableAt: DateTime.add(yield* DateTime.now, { milliseconds: delayMs }),
2238
+ },
2239
+ ],
2240
+ );
2241
+ });
2242
+
2243
// A hold waits for a person to review what they queued. Nobody reviews messages from agents,
2244
// the server or scheduled tasks, so a later completed run releases those unless it was stopped.
2245
const releaseUnattendedHolds = Effect.fn("Orchestrator.releaseUnattendedHolds")(function* (
@@ -5188,7 +5219,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
5219
const source = projection.runs.find((run) => run.id === command.restartContinuationOfRunId);
5220
if (
5221
!source ||
5191
- !isRestartContinuationSource(source, projection.providerTurns) ||
5222
+ !(
5223
+ isRestartContinuationSource(source, projection.providerTurns) ||
5224
+ continuationFailedBeforePrompt(source, projection.providerTurns)
5225
+ ) ||
5226
projection.thread.archivedAt !== null ||
5227
projection.thread.deletedAt !== null ||
5228
projection.thread.providerInstanceId !== source.providerInstanceId ||
@@ -10419,7 +10453,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
10453
// A resumed queue can have a lower ordinal than a pending continuation source.
10454
// Stop must reach those sources even when interrupting the resumed run succeeds.
10455
if (
10422
- run.status === "cancelled" &&
10456
+ (run.status === "cancelled" ||
10457
+ (run.status === "failed" && run.restartContinuationOfRunId !== undefined)) &&
10458
(yield* awaitsRestartContinuation(run).pipe(mapDispatchError(input.command)))
10459
) {
10460
yield* markStoppedRun({ ...input, thread, run });
@@ -13129,6 +13164,18 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
13164
// while writing child events, so nesting parent inside child would invert the order.
13165
// The keyed semaphores are neither reentrant nor deadlock-aware.
13166
if (delegatedTaskTerminalStatus(run.status) !== null) {
13167
+ // Before the parent hears of the failure: a pending continuation keeps the task open.
13168
+ if (run.status === "failed" && run.restartContinuationOfRunId !== undefined) {
13169
+ yield* threadDispatch.withLock(threadId, retryContinuationStart(threadId, run.id)).pipe(
13170
+ Effect.catchCause((cause) =>
13171
+ Effect.logWarning("Failed to schedule a restart continuation retry", {
13172
+ threadId,
13173
+ runId: run.id,
13174
+ cause,
13175
+ }),
13176
+ ),
13177
+ );
13178
+ }
13179
yield* threadDispatch.withLock(
13180
threadId,
13181
finalizeDelegatedCompletionDelivery(threadId, run.id),
apps/server/src/orchestration-v2/RestartBackgroundNote.ts
+45
-1
@@ -6,7 +6,7 @@ import type {
6
OrchestrationV2RunAttempt,
7
OrchestrationV2TurnItem,
8
} from "@t3tools/contracts";
9
-import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
9
+import { latestRootProviderFailure, runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
10
11
type Work = OrchestrationV2RestartCancelledBackgroundWork;
12
type Attempt = Pick<OrchestrationV2RunAttempt, "id" | "runId">;
@@ -181,6 +181,50 @@ export function cancelledBeforeStart(
181
return run.status === "cancelled" && neverStarted(run);
182
}
183
184
+/** The provider never accepted this continuation's prompt, so starting it again repeats nothing. */
185
+export function continuationFailedBeforePrompt(
186
+ run: Pick<OrchestrationV2Run, "status" | "activeAttemptId" | "restartContinuationOfRunId">,
187
+ providerTurns: ReadonlyArray<Pick<ProviderTurnState, "runAttemptId">>,
188
+): boolean {
189
+ return (
190
+ run.status === "failed" &&
191
+ run.restartContinuationOfRunId !== undefined &&
192
+ !providerTurns.some((turn) => turn.runAttemptId === run.activeAttemptId)
193
+ );
194
+}
195
+
196
+const CONTINUATION_START_RETRY_DELAYS_MS = [60_000, 180_000];
197
+
198
+/** Backoff before restarting a continuation that failed to start, or undefined to let it fail. */
199
+export function continuationStartRetryDelayMs(
200
+ run: OrchestrationV2Run,
201
+ projection: {
202
+ readonly runs: ReadonlyArray<OrchestrationV2Run>;
203
+ readonly providerTurns: ReadonlyArray<Pick<ProviderTurnState, "runAttemptId">>;
204
+ readonly turnItems: Parameters<typeof latestRootProviderFailure>[1];
205
+ },
206
+): number | undefined {
207
+ if (!continuationFailedBeforePrompt(run, projection.providerTurns)) return;
208
+ // These fail the same way however often they start.
209
+ const failureClass = latestRootProviderFailure(run, projection.turnItems)?.class;
210
+ if (
211
+ failureClass === "usage_limit" ||
212
+ failureClass === "permission_error" ||
213
+ failureClass === "validation_error"
214
+ )
215
+ return;
216
+ let failedStarts = 0;
217
+ for (
218
+ let current: OrchestrationV2Run | undefined = run;
219
+ current !== undefined && continuationFailedBeforePrompt(current, projection.providerTurns);
220
+ current = projection.runs.find(
221
+ (candidate) => candidate.id === current?.restartContinuationOfRunId,
222
+ )
223
+ )
224
+ failedStarts += 1;
225
+ return CONTINUATION_START_RETRY_DELAYS_MS[failedStarts - 1];
226
+}
227
+
228
export function isRestartContinuationSource(
229
source: Pick<OrchestrationV2Run, "status" | "activeAttemptId" | "restartCancelledBackgroundWork">,
230
providerTurns: ReadonlyArray<Pick<ProviderTurnState, "runAttemptId" | "status">>,
apps/server/src/orchestration-v2/RestartContinuation.test.ts
+120
-2
@@ -20,6 +20,7 @@ import * as Effect from "effect/Effect";
20
import * as Layer from "effect/Layer";
21
import * as ServerSettings from "../serverSettings.ts";
22
import { restartContinuationRun, continueRestartedRun } from "./RestartContinuation.ts";
23
+import { continuationStartRetryDelayMs } from "./RestartBackgroundNote.ts";
24
import * as ThreadManagementService from "./ThreadManagementService.ts";
25
import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts";
26
import * as ProjectionStore from "./ProjectionStore.ts";
@@ -46,6 +47,8 @@ function makeProjection() {
47
providerInstanceId: instanceId,
48
archivedAt: null,
49
deletedAt: null,
50
+ lineage: { parentThreadId: null, relationshipToParent: null },
51
+ forkedFrom: null,
52
},
53
runs: [
54
{
@@ -839,10 +842,13 @@ it.effect("does not cancel or resume a run that completes while shutdown intent
842
}),
843
);
844
842
-const continuationDispatches = (projection: OrchestrationV2ThreadProjection) =>
845
+const continuationDispatches = (
846
+ projection: OrchestrationV2ThreadProjection,
847
+ sourceRunId: RunId = runId,
848
+) =>
849
Effect.gen(function* () {
850
const dispatches: Array<Extract<OrchestrationV2Command, { type: "message.dispatch" }>> = [];
845
- yield* continueRestartedRun({ threadId, sourceRunId: runId }).pipe(
851
+ yield* continueRestartedRun({ threadId, sourceRunId }).pipe(
852
Effect.provide(
853
Layer.merge(
854
Layer.mock(ThreadManagementService.ThreadManagementService)({
@@ -943,6 +949,118 @@ it.effect("continues a resumed queued run that ran after an earlier continuation
949
}),
950
);
951
952
+it("backs off twice from a continuation the provider never took, then lets it fail", () => {
953
+ const base = makeProjection();
954
+ const failedStart = (id: string, of: RunId) => ({
955
+ ...base.runs[0]!,
956
+ id: RunId.make(id),
957
+ activeAttemptId: RunAttemptId.make(`attempt:${id}`),
958
+ status: "failed" as const,
959
+ restartContinuationOfRunId: of,
960
+ });
961
+ const first = failedStart("run:retry-1", runId);
962
+ const second = failedStart("run:retry-2", first.id);
963
+ const third = failedStart("run:retry-3", second.id);
964
+ const projection = (failureClass = "provider_error", providerTurns = base.providerTurns) =>
965
+ ({
966
+ runs: [{ ...base.runs[0]!, status: "cancelled" }, first, second, third],
967
+ providerTurns,
968
+ turnItems: [first, second, third].map((run) => ({
969
+ id: `error:${run.id}`,
970
+ type: "error",
971
+ status: "failed",
972
+ runId: run.id,
973
+ nodeId: run.rootNodeId,
974
+ ordinal: 1,
975
+ updatedAt: now,
976
+ failure: { class: failureClass, message: "Failed.", code: null, retryable: null },
977
+ })),
978
+ }) as unknown as Parameters<typeof continuationStartRetryDelayMs>[1];
979
+ assert.deepEqual(
980
+ [first, second, third].map((run) => continuationStartRetryDelayMs(run, projection())),
981
+ [60_000, 180_000, undefined],
982
+ );
983
+ for (const failureClass of ["usage_limit", "permission_error", "validation_error"])
984
+ assert.isUndefined(continuationStartRetryDelayMs(first, projection(failureClass)));
985
+ // Once the provider took the prompt, starting again would repeat the turn.
986
+ assert.isUndefined(
987
+ continuationStartRetryDelayMs(
988
+ first,
989
+ projection("provider_error", [
990
+ { ...base.providerTurns[0]!, runAttemptId: first.activeAttemptId, status: "failed" },
991
+ ] as never),
992
+ ),
993
+ );
994
+});
995
+
996
+it.effect("continues again after a continuation fails before its prompt", () =>
997
+ Effect.gen(function* () {
998
+ const cut = cutMidTurn();
999
+ const failedStart = {
1000
+ ...cut.runs[0]!,
1001
+ id: RunId.make("run:failed-start"),
1002
+ ordinal: 2,
1003
+ activeAttemptId: RunAttemptId.make("attempt:failed-start"),
1004
+ status: "failed",
1005
+ restartContinuationOfRunId: runId,
1006
+ };
1007
+ const dispatches = yield* continuationDispatches(
1008
+ { ...cut, runs: [...cut.runs, failedStart] } as unknown as OrchestrationV2ThreadProjection,
1009
+ failedStart.id,
1010
+ );
1011
+ assert.deepEqual(
1012
+ dispatches.map((command) => command.restartContinuationOfRunId),
1013
+ [failedStart.id],
1014
+ );
1015
+ }),
1016
+);
1017
+
1018
+it.effect.each([
1019
+ ["running", null, 0],
1020
+ ["completed", "Done.", 1],
1021
+] as const)(
1022
+ "a parent's Stop on the task's run blocks the child only while the task is %s",
1023
+ ([status, result, expected]) =>
1024
+ Effect.gen(function* () {
1025
+ const parentThreadId = ThreadId.make("thread:parent");
1026
+ const parentRunId = RunId.make("run:parent");
1027
+ const taskId = NodeId.make("node:task");
1028
+ const cut = cutMidTurn();
1029
+ const child = {
1030
+ ...cut,
1031
+ thread: {
1032
+ ...cut.thread,
1033
+ lineage: { parentThreadId, relationshipToParent: "subagent" },
1034
+ forkedFrom: { type: "node", nodeId: taskId },
1035
+ },
1036
+ } as unknown as OrchestrationV2ThreadProjection;
1037
+ const parent = {
1038
+ thread: { id: parentThreadId },
1039
+ subagents: [{ id: taskId, runId: parentRunId, origin: "app_owned", status, result }],
1040
+ turnItems: [{ id: "turn-item:stop", runId: parentRunId, type: "run_interrupt_request" }],
1041
+ };
1042
+ let dispatched = 0;
1043
+ yield* continueRestartedRun({ threadId, sourceRunId: runId }).pipe(
1044
+ Effect.provide(
1045
+ Layer.merge(
1046
+ Layer.mock(ThreadManagementService.ThreadManagementService)({
1047
+ getThreadRecords: (id) =>
1048
+ Effect.succeed((id === parentThreadId ? parent : child) as never),
1049
+ recoverDelegatedTask: () => Effect.void,
1050
+ dispatch: () =>
1051
+ Effect.sync(() => {
1052
+ dispatched += 1;
1053
+ return {} as never;
1054
+ }),
1055
+ }),
1056
+ ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
1057
+ ),
1058
+ ),
1059
+ );
1060
+ assert.equal(dispatched, expected);
1061
+ }),
1062
+);
1063
+
1064
it.effect("does not continue a run the user asked to stop before the restart", () =>
1065
Effect.gen(function* () {
1066
const texts = yield* continuationTexts({
apps/server/src/orchestration-v2/RestartContinuation.ts
+32
-1
@@ -2,6 +2,7 @@ import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
2
import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
3
import {
4
CommandId,
5
+ isOrchestrationV2WorkActive,
6
latestProviderTurnForAttempt,
7
MessageId,
8
type OrchestrationV2Run,
@@ -19,6 +20,7 @@ import {
20
awaitingProviderThreadId,
21
cancelledRosterTaskWork,
22
cancelledTurnItemWork,
23
+ continuationFailedBeforePrompt,
24
isAwaitedWork,
25
isRestartContinuationSource,
26
cancelledBeforeStart,
@@ -200,7 +202,10 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta
202
const source = projection.runs.find((run) => run.id === input.sourceRunId);
203
if (
204
!source ||
203
- !isRestartContinuationSource(source, projection.providerTurns) ||
205
+ !(
206
+ isRestartContinuationSource(source, projection.providerTurns) ||
207
+ continuationFailedBeforePrompt(source, projection.providerTurns)
208
+ ) ||
209
restartContinuationBlocked(
210
projection,
211
source,
@@ -255,6 +260,32 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta
260
)
261
)
262
return;
263
+ // A parent's Stop reaches its open delegated tasks through an effect a restart can cut short.
264
+ const { lineage, forkedFrom } = projection.thread;
265
+ const taskId =
266
+ source.delegatedTaskId ?? (forkedFrom?.type === "node" ? forkedFrom.nodeId : undefined);
267
+ if (
268
+ lineage.relationshipToParent === "subagent" &&
269
+ lineage.parentThreadId !== null &&
270
+ taskId !== undefined
271
+ ) {
272
+ const task = (yield* threads.getThreadRecords(lineage.parentThreadId, ["subagents"], {
273
+ subagentIds: [taskId],
274
+ })).subagents.find((candidate) => candidate.id === taskId);
275
+ // A settled task no longer owns later work in its thread.
276
+ const parentRunId =
277
+ task !== undefined && task.result === null && isOrchestrationV2WorkActive(task.status)
278
+ ? task.runId
279
+ : null;
280
+ if (
281
+ parentRunId !== null &&
282
+ (yield* threads.getThreadRecords(lineage.parentThreadId, ["turnItems"], {
283
+ turnItemTypes: ["run_interrupt_request"],
284
+ turnItemRunIds: [parentRunId],
285
+ })).turnItems.length > 0
286
+ )
287
+ return;
288
+ }
289
const sourceMessage = sourceRecords.messages.find(
290
(message) => message.id === source.userMessageId,
291
);
apps/server/src/orchestration-v2/runtimeLayer.test.ts
+159
@@ -5465,6 +5465,165 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
5465
}),
5466
);
5467
5468
+ it.effect.each([false, true])(
5469
+ "restarts a continuation that failed before its prompt, unless stopped (%s)",
5470
+ (stopped) =>
5471
+ Effect.gen(function* () {
5472
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
5473
+ const eventSink = yield* EventSink.EventSinkV2;
5474
+ const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
5475
+ const threadId = ThreadId.make(`runtime-layer-continuation-start-retry-${stopped}`);
5476
+ const messageId = (key: string) => MessageId.make(`${threadId}:${key}`);
5477
+ yield* orchestrator.dispatch({
5478
+ type: "thread.create",
5479
+ createdBy: "user",
5480
+ creationSource: "web",
5481
+ commandId: CommandId.make(`${threadId}:create`),
5482
+ threadId,
5483
+ projectId: ProjectId.make(`${threadId}:project`),
5484
+ title: "Continuation start retry",
5485
+ modelSelection,
5486
+ runtimeMode: "full-access",
5487
+ interactionMode: "default",
5488
+ branch: null,
5489
+ worktreePath: process.cwd(),
5490
+ });
5491
+ const runFor = (key: string) =>
5492
+ Effect.map(orchestrator.getThreadProjection(threadId), ({ runs }) =>
5493
+ runs.find((candidate) => candidate.userMessageId === messageId(key)),
5494
+ );
5495
+ const write = (
5496
+ key: string,
5497
+ payload: OrchestrationV2Run,
5498
+ extra: Parameters<typeof eventSink.write>[0]["events"] = [],
5499
+ ) =>
5500
+ Effect.gen(function* () {
5501
+ const now = yield* DateTime.now;
5502
+ yield* eventSink.write({
5503
+ events: [
5504
+ ...extra,
5505
+ {
5506
+ id: EventId.make(`${threadId}:${key}`),
5507
+ type: "run.updated",
5508
+ threadId,
5509
+ runId: payload.id,
5510
+ providerInstanceId: payload.providerInstanceId,
5511
+ occurredAt: now,
5512
+ payload: { ...payload, startedAt: now, completedAt: now },
5513
+ },
5514
+ ],
5515
+ });
5516
+ });
5517
+ yield* orchestrator.dispatch({
5518
+ type: "message.dispatch",
5519
+ createdBy: "user",
5520
+ creationSource: "web",
5521
+ commandId: CommandId.make(`${threadId}:message:cut`),
5522
+ threadId,
5523
+ messageId: messageId("cut"),
5524
+ text: "Work that the restart cut",
5525
+ attachments: [],
5526
+ modelSelection,
5527
+ dispatchMode: { type: "start_immediately" },
5528
+ });
5529
+ const cut = (yield* runFor("cut"))!;
5530
+ yield* write("cut:cancelled", { ...cut, status: "cancelled" });
5531
+ yield* orchestrator.dispatch({
5532
+ type: "message.dispatch",
5533
+ createdBy: "agent",
5534
+ creationSource: "server",
5535
+ commandId: CommandId.make(`${threadId}:message:continuation`),
5536
+ threadId,
5537
+ messageId: messageId("continuation"),
5538
+ text: "Continue where you left off.",
5539
+ attachments: [],
5540
+ modelSelection,
5541
+ dispatchMode: { type: "start_immediately" },
5542
+ restartContinuationOfRunId: cut.id,
5543
+ });
5544
+ yield* orchestrator.dispatch({
5545
+ type: "message.dispatch",
5546
+ createdBy: "user",
5547
+ creationSource: "web",
5548
+ commandId: CommandId.make(`${threadId}:message:queued`),
5549
+ threadId,
5550
+ messageId: messageId("queued"),
5551
+ text: "Queued behind the continuation",
5552
+ attachments: [],
5553
+ modelSelection,
5554
+ dispatchMode: { type: "queue_after_active" },
5555
+ });
5556
+ const continuation = (yield* runFor("continuation"))!;
5557
+ assert.isNotNull(continuation.rootNodeId);
5558
+ const queued = (yield* runFor("queued"))!;
5559
+ // The failure holds the queue after the reactor scheduled any retry.
5560
+ const held = yield* eventSink
5561
+ .stream({ threadId, afterSequence: yield* orchestrator.getThreadEventSequence(threadId) })
5562
+ .pipe(
5563
+ Stream.filter(
5564
+ (stored) =>
5565
+ stored.event.type === "run.updated" &&
5566
+ stored.event.payload.id === queued.id &&
5567
+ stored.event.payload.queueHeld === true,
5568
+ ),
5569
+ Stream.runHead,
5570
+ Effect.forkScoped,
5571
+ );
5572
+ yield* Effect.yieldNow;
5573
+ const now = yield* DateTime.now;
5574
+ yield* write("continuation:failed", { ...continuation, status: "failed" }, [
5575
+ {
5576
+ id: EventId.make(`${threadId}:continuation:error`),
5577
+ type: "turn-item.updated",
5578
+ threadId,
5579
+ runId: continuation.id,
5580
+ nodeId: continuation.rootNodeId,
5581
+ providerInstanceId: continuation.providerInstanceId,
5582
+ occurredAt: now,
5583
+ payload: {
5584
+ id: TurnItemId.make(`${threadId}:continuation:error`),
5585
+ type: "error",
5586
+ threadId,
5587
+ runId: continuation.id,
5588
+ nodeId: continuation.rootNodeId,
5589
+ providerThreadId: continuation.providerThreadId,
5590
+ providerTurnId: null,
5591
+ nativeItemRef: null,
5592
+ parentItemId: null,
5593
+ ordinal: 2,
5594
+ status: "failed",
5595
+ title: "Provider failure",
5596
+ startedAt: now,
5597
+ completedAt: now,
5598
+ updatedAt: now,
5599
+ failure: {
5600
+ class: "provider_error",
5601
+ message: "Claude timed out while registering its MCP tools.",
5602
+ code: null,
5603
+ retryable: null,
5604
+ },
5605
+ },
5606
+ },
5607
+ ]);
5608
+ yield* Fiber.join(held);
5609
+ const retriesOf = Effect.map(orchestrator.getThreadProjection(threadId), ({ runs }) =>
5610
+ runs.filter((run) => run.restartContinuationOfRunId === continuation.id),
5611
+ );
5612
+ yield* worker.drain();
5613
+ assert.deepEqual(yield* retriesOf, [], "the retry waits out its backoff");
5614
+ if (stopped) {
5615
+ yield* orchestrator.dispatch({
5616
+ type: "thread.stop",
5617
+ commandId: CommandId.make(`${threadId}:stop`),
5618
+ threadId,
5619
+ });
5620
+ }
5621
+ yield* TestClock.adjust("61 seconds");
5622
+ yield* worker.drain();
5623
+ assert.lengthOf(yield* retriesOf, stopped ? 0 : 1);
5624
+ }),
5625
+ );
5626
+
5627
it.effect.each(["startup", "shutdown"] as const)(
5628
"preserves and holds queued messages across %s until explicitly resumed",
5629
(trigger) =>