The best way to control Claude Code, Codex and more from any computer or phone
fix(server): wait for automatic follow-ups before settling threads (#31815)
Keep deferred settlement requests through automatic follow-up turns and retry from persisted run events, including replayed rejected commands. Preserve failed follow-ups and new user input, and guard settlement against newer runs. Validated with 173 focused tests through check:changed, all CI checks on e1a41f6, and resolved agentic reviews. Model: GPT 6.1 Sol. Harness: Codex in T3 Code.
Yash Singh committed
Nov 5, 2026 at 23:30 UTC
6f54322e4a57e38c01fabf4e5da0d51fe3a24270
5 files changed
+219
-20
apps/server/src/orchestration-v2/Orchestrator.ts
+13
@@ -3454,6 +3454,19 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
3454
}
3455
if (command.type === "thread.settle") {
3456
const projection = yield* loadProjectionForCommand(command, ["runs", "runtimeRequests"]);
3457
+ if (
3458
+ command.expectedLatestRunId !== undefined &&
3459
+ projection.runs.reduce<OrchestrationV2Run | undefined>(
3460
+ (latest, run) => (latest === undefined || run.ordinal > latest.ordinal ? run : latest),
3461
+ undefined,
3462
+ )?.id !== command.expectedLatestRunId
3463
+ ) {
3464
+ return yield* new OrchestratorDispatchError({
3465
+ commandId: command.commandId,
3466
+ commandType: command.type,
3467
+ cause: "The thread received another run before settlement.",
3468
+ });
3469
+ }
3470
const queuedMessages = yield* projectionStore
3471
.getMessagesById(
3472
command.threadId,
apps/server/src/orchestration-v2/ThreadManagementService.test.ts
+118
-14
@@ -559,28 +559,108 @@ it.effect.each(["live", "no-session", "unsupported-runtime"] as const)(
559
);
560
561
it.effect.each([
562
- { status: "completed" as const, settles: true },
563
- { status: "failed" as const, settles: false },
564
- { status: "interrupted" as const, settles: false },
565
-])("settleAfterRun settles only when the run $status", ({ status, settles }) =>
562
+ { status: "completed" as const, followUp: "none", settles: true },
563
+ { status: "failed" as const, followUp: "none", settles: false },
564
+ { status: "interrupted" as const, followUp: "none", settles: false },
565
+ { status: "completed" as const, followUp: "notification", settles: true },
566
+ { status: "completed" as const, followUp: "replayed-notification", settles: true },
567
+ { status: "completed" as const, followUp: "delegated", settles: true },
568
+ { status: "completed" as const, followUp: "failed-notification", settles: false },
569
+ { status: "completed" as const, followUp: "held-notification", settles: true },
570
+ { status: "completed" as const, followUp: "user", settles: false },
571
+ { status: "completed" as const, followUp: "steered-notification", settles: false },
572
+])("settleAfterRun handles $status with $followUp follow-up", ({ status, followUp, settles }) =>
573
Effect.gen(function* () {
574
const projectId = ProjectId.make("project:thread-management:settle-after-run");
575
const threadId = ThreadId.make("thread:thread-management:settle-after-run");
576
const runId = RunId.make("run:thread-management:settle-after-run");
577
+ const followUpId = RunId.make(`${runId}:follow-up`);
578
+ const subscribed = yield* Deferred.make<void>();
579
+ const events = yield* Queue.unbounded<OrchestrationV2StoredEvent>();
580
+ const replayRejected = yield* Deferred.make<void>();
581
+ let sequence = 0;
582
+ let sequenceReads = 0;
583
+ let rejectedCommandId: CommandId | undefined;
584
+ const runs = [{ id: runId, ordinal: 1, status, userMessageId: MessageId.make("original") }];
585
+ const nextRun = {
586
+ id: followUpId,
587
+ ordinal: 2,
588
+ status: "running" as OrchestrationV2Run["status"],
589
+ userMessageId: MessageId.make("follow-up"),
590
+ };
591
const dispatched: Array<string> = [];
592
+ // The notification starts after the completed run was observed, before settlement commits.
593
+ let raced = false;
594
const layerTest = ThreadManagementService.layer.pipe(
595
Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})),
596
Layer.provide(
597
Layer.mock(Orchestrator.OrchestratorV2)({
575
- getThreadEventSequence: () => Effect.succeed(0),
598
+ getThreadEventSequence: () =>
599
+ Effect.sync(() => {
600
+ if (followUp === "replayed-notification" && sequenceReads++ === 2) sequence = 1;
601
+ return sequence;
602
+ }),
603
restartContinuationPending: () => Effect.succeed(false),
577
- getThreadRecords: () =>
578
- Effect.succeed({
579
- thread: { id: threadId, projectId, deletedAt: null },
580
- runs: [{ id: runId, status }],
581
- } as unknown as OrchestrationV2ThreadProjection),
604
+ streamStoredEventsFrom: () =>
605
+ Stream.fromEffect(Deferred.succeed(subscribed, undefined)).pipe(
606
+ Stream.drain,
607
+ Stream.concat(Stream.fromQueue(events)),
608
+ ),
609
+ getThreadRecords: (_threadId, _fields, filter) =>
610
+ Effect.sync(
611
+ () =>
612
+ ({
613
+ thread: { id: threadId, projectId, deletedAt: null },
614
+ runs: raced ? [...runs, nextRun] : runs,
615
+ messages: [
616
+ { id: MessageId.make("original"), runId, role: "user" as const },
617
+ {
618
+ id: nextRun.userMessageId,
619
+ runId: followUpId,
620
+ role: "user" as const,
621
+ ...(followUp.endsWith("notification") ? { notification: {} } : {}),
622
+ ...(followUp === "delegated" ? { delegatedCompletion: {} } : {}),
623
+ },
624
+ { id: MessageId.make("reply"), runId: followUpId, role: "assistant" as const },
625
+ ...(followUp === "steered-notification"
626
+ ? [
627
+ {
628
+ id: MessageId.make("user-steer"),
629
+ runId: followUpId,
630
+ role: "user" as const,
631
+ },
632
+ ]
633
+ : []),
634
+ ].filter(
635
+ (message) =>
636
+ (filter?.messageIds === undefined ||
637
+ filter.messageIds.includes(message.id)) &&
638
+ (filter?.messageRunIds === undefined ||
639
+ filter.messageRunIds.includes(message.runId)) &&
640
+ (filter?.messageRoles === undefined ||
641
+ filter.messageRoles.includes(message.role)),
642
+ ),
643
+ }) as unknown as OrchestrationV2ThreadProjection,
644
+ ),
645
dispatch: (command) =>
583
- Effect.sync(() => {
646
+ Effect.gen(function* () {
647
+ if (followUp !== "none" && !raced) {
648
+ raced = true;
649
+ rejectedCommandId = command.commandId;
650
+ return yield* new Orchestrator.OrchestratorDispatchError({
651
+ commandId: command.commandId,
652
+ commandType: command.type,
653
+ cause: "Thread is still running",
654
+ });
655
+ }
656
+ if (followUp === "replayed-notification" && command.commandId === rejectedCommandId) {
657
+ yield* Deferred.succeed(replayRejected, undefined);
658
+ return yield* new Orchestrator.OrchestratorCommandPreviouslyRejectedError({
659
+ commandId: command.commandId,
660
+ commandType: command.type,
661
+ detail: "Thread is still running",
662
+ });
663
+ }
664
dispatched.push(command.type);
665
return { sequence: 1, storedEvents: [] } as never;
666
}),
@@ -592,9 +672,33 @@ it.effect.each([
672
const service = yield* ThreadManagementService.ThreadManagementService.pipe(
673
Effect.provide(layerTest),
674
);
595
-
596
- yield* service.settleAfterRun({ projectId, threadId, runId });
597
-
675
+ const fiber = yield* service
676
+ .settleAfterRun({ projectId, threadId, runId })
677
+ .pipe(Effect.forkChild);
678
+ if (followUp !== "none") {
679
+ yield* Deferred.await(subscribed);
680
+ expect(dispatched).toEqual([]);
681
+ nextRun.status =
682
+ followUp === "failed-notification"
683
+ ? "failed"
684
+ : followUp === "held-notification"
685
+ ? "queued"
686
+ : "completed";
687
+ yield* Queue.offer(events, {
688
+ sequence: 1,
689
+ event: { type: "run.updated", threadId, payload: nextRun },
690
+ } as unknown as OrchestrationV2StoredEvent);
691
+ }
692
+ if (followUp === "replayed-notification") {
693
+ yield* Deferred.await(replayRejected);
694
+ expect(dispatched).toEqual([]);
695
+ sequence = 2;
696
+ yield* Queue.offer(events, {
697
+ sequence,
698
+ event: { type: "run.updated", threadId, payload: nextRun },
699
+ } as unknown as OrchestrationV2StoredEvent);
700
+ }
701
+ yield* Fiber.join(fiber);
702
expect(dispatched).toEqual(settles ? ["thread.settle"] : []);
703
}),
704
);
apps/server/src/orchestration-v2/ThreadManagementService.ts
+69
-6
@@ -446,8 +446,8 @@ export interface ThreadManagementServiceShape {
446
readonly waitForThread: (
447
input: ThreadManagementWaitInput,
448
) => Effect.Effect<ThreadManagementWaitResult, ThreadManagementError>;
449
- /** Waits for the agent's run to end before settling its thread and stopping the session.
450
- * Other endings or settle refusals leave the thread active. */
449
+ /** Waits for the agent's run and automatic follow-ups before settling.
450
+ * A new user turn or an unsuccessful ending leaves the thread active. */
451
readonly settleAfterRun: (input: {
452
readonly projectId: ProjectId;
453
readonly threadId: ThreadId;
@@ -1044,11 +1044,74 @@ const make = Effect.gen(function* () {
1044
)(function* (input) {
1045
const { run } = yield* waitForThread({ ...input, timeoutMs: SETTLE_AFTER_RUN_WAIT_MS });
1046
if (run?.status !== "completed") return;
1047
- yield* dispatch({
1048
- type: "thread.settle",
1049
- commandId: CommandId.make(`server:settle-after-run:${input.runId}`),
1050
- threadId: input.threadId,
1047
+ const afterSequence = yield* orchestrator.getThreadEventSequence(input.threadId);
1048
+ const trySettle = Effect.gen(function* () {
1049
+ const sequence = yield* orchestrator.getThreadEventSequence(input.threadId);
1050
+ const records = yield* getProjectThreadRecords(input, ["runs"]);
1051
+ const following = records.runs.filter((candidate) => candidate.ordinal > run.ordinal);
1052
+ if (following.length > 0) {
1053
+ const { messages } = yield* getProjectThreadRecords(input, ["messages"], {
1054
+ messageRunIds: following.map((candidate) => candidate.id),
1055
+ messageRoles: ["user"],
1056
+ });
1057
+ const automaticMessageIds = new Set(
1058
+ messages
1059
+ .filter(
1060
+ (message) =>
1061
+ message.role === "user" &&
1062
+ (message.notification !== undefined || message.delegatedCompletion !== undefined),
1063
+ )
1064
+ .map((message) => message.id),
1065
+ );
1066
+ if (
1067
+ messages.some((message) => !automaticMessageIds.has(message.id)) ||
1068
+ following.some((candidate) => !automaticMessageIds.has(candidate.userMessageId))
1069
+ ) {
1070
+ return true;
1071
+ }
1072
+ if (
1073
+ following.some(
1074
+ (candidate) => candidate.status !== "queued" && !isTerminalRunStatus(candidate.status),
1075
+ )
1076
+ )
1077
+ return false;
1078
+ if (
1079
+ following.some(
1080
+ (candidate) => candidate.status !== "queued" && candidate.status !== "completed",
1081
+ )
1082
+ )
1083
+ return true;
1084
+ }
1085
+ const latest = latestRun(records);
1086
+ if (latest === undefined) return true;
1087
+ return yield* dispatch({
1088
+ type: "thread.settle",
1089
+ commandId: CommandId.make(`server:settle-after-run:${input.runId}:${sequence}`),
1090
+ threadId: input.threadId,
1091
+ expectedLatestRunId: latest.id,
1092
+ }).pipe(
1093
+ Effect.as(true),
1094
+ // A notification can start between the projection read and the guarded dispatch.
1095
+ Effect.catchTags({
1096
+ OrchestratorDispatchError: () => Effect.succeed(false),
1097
+ OrchestratorCommandPreviouslyRejectedError: () => Effect.succeed(false),
1098
+ }),
1099
+ );
1100
});
1101
+ yield* Stream.fromEffect(trySettle).pipe(
1102
+ Stream.concat(
1103
+ orchestrator
1104
+ .streamStoredEventsFrom({
1105
+ threadId: input.threadId,
1106
+ afterSequence,
1107
+ eventType: "run.updated",
1108
+ })
1109
+ .pipe(Stream.mapEffect(() => trySettle)),
1110
+ ),
1111
+ Stream.filter((done) => done),
1112
+ Stream.runHead,
1113
+ Effect.timeoutOption(Duration.millis(SETTLE_AFTER_RUN_WAIT_MS)),
1114
+ );
1115
});
1116
1117
const settleThread: ThreadManagementServiceShape["settleThread"] = Effect.fn(
apps/server/src/orchestration-v2/runtimeLayer.test.ts
+18
@@ -4859,11 +4859,29 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
4859
});
4860
});
4861
yield* holdQueueAfterRestart("settle-automatic-restart");
4862
+ const beforeSettle = yield* orchestrator.getThreadProjection(threadId);
4863
+ const oldestRun = beforeSettle.runs.reduce((oldest, run) =>
4864
+ run.ordinal < oldest.ordinal ? run : oldest,
4865
+ );
4866
+ const latestRun = beforeSettle.runs.reduce((latest, run) =>
4867
+ run.ordinal > latest.ordinal ? run : latest,
4868
+ );
4869
+ const staleSettle = yield* orchestrator
4870
+ .dispatch({
4871
+ type: "thread.settle",
4872
+ commandId: CommandId.make("settle-automatic-stale"),
4873
+ threadId,
4874
+ expectedLatestRunId: oldestRun.id,
4875
+ })
4876
+ .pipe(Effect.flip);
4877
+ assert.equal(staleSettle._tag, "OrchestratorDispatchError");
4878
+ assert.isNull((yield* orchestrator.getThreadProjection(threadId)).thread.settledAt);
4879
4880
yield* orchestrator.dispatch({
4881
type: "thread.settle",
4882
commandId: CommandId.make("settle-automatic-settle"),
4883
threadId,
4884
+ expectedLatestRunId: latestRun.id,
4885
});
4886
const settled = yield* orchestrator.getThreadProjection(threadId);
4887
assert.isNotNull(settled.thread.settledAt);
packages/contracts/src/orchestrationV2.ts
+1
@@ -2975,6 +2975,7 @@ export const OrchestrationV2Command = Schema.Union([
2975
type: Schema.Literal("thread.settle"),
2976
commandId: CommandId,
2977
threadId: ThreadId,
2978
+ expectedLatestRunId: Schema.optional(RunId),
2979
settledAt: Schema.optional(Schema.DateTimeUtc),
2980
}),
2981
/** Rejects changes since snapshotAt or any explicit settled override to preserve user actions. */