@devries-cameron20 / t3code-1 / commits / 867112cda0
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): agent questions stay answerable after a server restart (#19229)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Julius Marminge <julius0216@outlook.com>

Dara Adedeji committed Oct 27, 2026 at 19:11 UTC 867112cda0906960f6340171c5424600a1ac1d28
7 files changed +718 -147
apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts
+172 -1
@@ -15,11 +15,14 @@ import {
15 ProviderInstanceId,
16 ProviderThreadId,
17 RunId,
18 + RuntimeRequestId,
19 ThreadId,
20 TurnItemId,
21 } from "@t3tools/contracts";
22 import * as DateTime from "effect/DateTime";
23 import * as Effect from "effect/Effect";
24 +import * as Deferred from "effect/Deferred";
25 +import * as Fiber from "effect/Fiber";
26 import * as Layer from "effect/Layer";
27 import * as Stream from "effect/Stream";
28
@@ -40,6 +43,7 @@ import * as EventSink from "./EventSink.ts";
43 import * as Orchestrator from "./Orchestrator.ts";
44 import { continueRestartedRun } from "./RestartContinuation.ts";
45 import * as RuntimeLayer from "./runtimeLayer.ts";
46 +import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
47 import * as ProviderTurnStartServiceTestkit from "./ProviderTurnStartService.testkit.ts";
48 import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
49 import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests";
@@ -104,7 +108,11 @@ const layerTestProviderInstanceRegistry = Layer.succeed(
108 },
109 );
110
107 -const layerTest = Layer.mergeAll(RuntimeLayer.layer, RuntimeLayer.layerEventSink).pipe(
111 +const layerTest = Layer.mergeAll(
112 + RuntimeLayer.layer,
113 + RuntimeLayer.layerEventSink,
114 + ThreadCommandExecutor.layer,
115 +).pipe(
116 Layer.provideMerge(RuntimeLayer.layerProjectService),
117 Layer.provide(
118 Layer.mock(WorkspacePaths.WorkspacePaths)({
@@ -1980,6 +1988,169 @@ it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))(
1988 }
1989 }),
1990 );
1991 +
1992 + it.effect("serializes a recovered child answer with its parent and tracks the follow-up", () =>
1993 + Effect.gen(function* () {
1994 + const orchestrator = yield* Orchestrator.OrchestratorV2;
1995 + const eventSink = yield* EventSink.EventSinkV2;
1996 + const now = yield* DateTime.now;
1997 + const parentThreadId = ThreadId.make("thread:question-answer-parent");
1998 + const projectId = ProjectId.make("project:question-answer");
1999 + const parentRunId = RunId.make("run:question-answer-parent");
2000 + const rootNodeId = NodeId.make("node:question-answer-parent");
2001 + yield* seedParentWithTerminalTask({
2002 + threadId: parentThreadId,
2003 + projectId,
2004 + runId: parentRunId,
2005 + rootNodeId,
2006 + taskId: NodeId.make("node:question-answer-seed"),
2007 + deliveryState: "acknowledged",
2008 + now,
2009 + });
2010 + const child = yield* seedRestartCancelledChild({
2011 + parentThreadId,
2012 + projectId,
2013 + parentRunId,
2014 + rootNodeId,
2015 + name: "question-answer-child",
2016 + completionWake: "always",
2017 + continuationPending: false,
2018 + now,
2019 + });
2020 + // The cancelled result is published before anyone answers the question.
2021 + yield* orchestrator.recoverDelegatedTasks;
2022 + assert.isNotNull(
2023 + (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find(
2024 + (task) => task.id === child.taskId,
2025 + )?.result,
2026 + );
2027 + const parentRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs.find(
2028 + (run) => run.id === parentRunId,
2029 + )!;
2030 + yield* eventSink.write({
2031 + events: [
2032 + {
2033 + id: EventId.make("event:question-answer-parent-completed"),
2034 + type: "run.updated",
2035 + threadId: parentThreadId,
2036 + runId: parentRunId,
2037 + occurredAt: now,
2038 + payload: { ...parentRun, status: "completed", completedAt: now },
2039 + },
2040 + ],
2041 + });
2042 + const requestId = RuntimeRequestId.make("request:question-answer");
2043 + const nodeId = NodeId.make("node:question-answer-request");
2044 + yield* eventSink.write({
2045 + commandId: CommandId.make("command:seed-question-answer"),
2046 + events: [
2047 + {
2048 + id: EventId.make("event:question-answer-request"),
2049 + type: "runtime-request.updated",
2050 + threadId: child.childThreadId,
2051 + nodeId,
2052 + occurredAt: now,
2053 + payload: {
2054 + id: requestId,
2055 + nodeId,
2056 + providerTurnId: null,
2057 + nativeRequestRef: null,
2058 + kind: "user_input",
2059 + status: "pending",
2060 + responseCapability: { type: "message" },
2061 + createdAt: now,
2062 + resolvedAt: null,
2063 + },
2064 + },
2065 + {
2066 + id: EventId.make("event:question-answer-item"),
2067 + type: "turn-item.updated",
2068 + threadId: child.childThreadId,
2069 + nodeId,
2070 + occurredAt: now,
2071 + payload: {
2072 + id: TurnItemId.make("turn-item:question-answer"),
2073 + type: "user_input_request",
2074 + threadId: child.childThreadId,
2075 + runId: child.childRunId,
2076 + nodeId,
2077 + providerThreadId: null,
2078 + providerTurnId: null,
2079 + nativeItemRef: null,
2080 + parentItemId: null,
2081 + ordinal: 0,
2082 + status: "waiting",
2083 + title: null,
2084 + startedAt: now,
2085 + completedAt: null,
2086 + updatedAt: now,
2087 + requestId,
2088 + questions: [
2089 + { id: "color", header: "Color", question: "Which color?", options: [] },
2090 + ],
2091 + },
2092 + },
2093 + ],
2094 + });
2095 +
2096 + const executor = yield* ThreadCommandExecutor.ThreadCommandExecutor;
2097 + const parentHeld = yield* Deferred.make<void>();
2098 + const releaseParent = yield* Deferred.make<void>();
2099 + const answerWaiting = yield* Deferred.make<void>();
2100 + const withLock = executor.withLock;
2101 + const parentMutation = yield* withLock(
2102 + parentThreadId,
2103 + Deferred.succeed(parentHeld, undefined).pipe(
2104 + Effect.andThen(Deferred.await(releaseParent)),
2105 + ),
2106 + ).pipe(Effect.forkScoped({ startImmediately: true }));
2107 + yield* Deferred.await(parentHeld);
2108 + const observeLock: ThreadCommandExecutor.ThreadCommandExecutor["Service"]["withLock"] = (
2109 + key,
2110 + effect,
2111 + ) =>
2112 + key === parentThreadId
2113 + ? Deferred.succeed(answerWaiting, undefined).pipe(Effect.andThen(withLock(key, effect)))
2114 + : withLock(key, effect);
2115 + const spy = vi.spyOn(executor, "withLock").mockImplementation(observeLock);
2116 + yield* Effect.gen(function* () {
2117 + const answer = yield* orchestrator
2118 + .dispatch({
2119 + type: "runtime-request.respond",
2120 + commandId: CommandId.make("command:question-answer"),
2121 + threadId: child.childThreadId,
2122 + requestId,
2123 + answers: { color: "Blue" },
2124 + })
2125 + .pipe(Effect.forkScoped({ startImmediately: true }));
2126 + yield* Effect.raceFirst(Deferred.await(answerWaiting), Fiber.join(answer));
2127 + const waiting = yield* orchestrator.getThreadProjection(child.childThreadId);
2128 + assert.equal(
2129 + waiting.runtimeRequests.find((row) => row.id === requestId)?.status,
2130 + "pending",
2131 + );
2132 + assert.isFalse((yield* executor.activeKeys).includes(child.childThreadId));
2133 + yield* Deferred.succeed(releaseParent, undefined);
2134 + yield* Fiber.join(parentMutation);
2135 + yield* Fiber.join(answer);
2136 + }).pipe(Effect.ensuring(Effect.sync(() => spy.mockRestore())));
2137 +
2138 + const answerRun = (yield* orchestrator.getThreadProjection(child.childThreadId)).runs.find(
2139 + (run) => run.userMessageId === MessageId.make(`async-answer:${requestId}`),
2140 + );
2141 + assert.isDefined(answerRun?.delegatedTaskId);
2142 + const followup = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find(
2143 + (task) => task.id === answerRun?.delegatedTaskId,
2144 + );
2145 + assert.equal(followup?.childThreadId, child.childThreadId);
2146 + assert.isNull(followup?.result);
2147 + assert.isTrue(
2148 + (yield* orchestrator.getThreadShell(parentThreadId))?.pendingBackgroundTasks?.some(
2149 + (task) => task.kind === "subagent" && task.childThreadId === child.childThreadId,
2150 + ),
2151 + );
2152 + }),
2153 + );
2154 },
2155 );
2156
apps/server/src/orchestration-v2/Orchestrator.ts
+95 -48
@@ -62,12 +62,14 @@ import {
62 ProviderInstanceId,
63 ProviderInteractionMode,
64 resolveProviderInstanceEnabled,
65 + resolveProjectProviderInstanceEnabled,
66 type ProviderSessionId,
67 RunId,
68 RuntimeMode,
69 ThreadLinkedPullRequest,
70 ThreadId,
71 TurnItemId,
72 + TrimmedNonEmptyString,
73 } from "@t3tools/contracts";
74 import { modelSelectionsEqual } from "@t3tools/shared/model";
75 import { recordThreadOpen } from "@t3tools/shared/paletteSearch";
@@ -167,6 +169,10 @@ import { planThreadDeletion } from "./ThreadDeletion.ts";
169 import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
170 import * as ServerSettings from "../serverSettings.ts";
171
172 +const isMessageQuestionAnswer = Schema.is(
173 + Schema.Union([TrimmedNonEmptyString, Schema.NonEmptyArray(TrimmedNonEmptyString)]),
174 +);
175 +
176 export class OrchestratorDispatchError extends Schema.TaggedError<OrchestratorDispatchError>()(
177 "OrchestratorDispatchError",
178 {
@@ -8517,6 +8523,77 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
8523 );
8524 }
8525
8526 + const approvalTurnItem = context.item;
8527 + const resolvedNodeStatus =
8528 + command.decision === "decline" || command.decision === "cancel"
8529 + ? ("cancelled" as const)
8530 + : ("completed" as const);
8531 + const replies: string[] = [];
8532 + if (
8533 + runtimeRequest.responseCapability.type === "message" &&
8534 + resolvedNodeStatus !== "cancelled"
8535 + ) {
8536 + if (approvalTurnItem?.type !== "user_input_request") {
8537 + return yield* new OrchestratorDispatchError({
8538 + commandId: command.commandId,
8539 + commandType: command.type,
8540 + cause: "The question for this request was not found.",
8541 + });
8542 + }
8543 + const thread = yield* projectionStore
8544 + .getThread(command.threadId)
8545 + .pipe(mapDispatchError(command));
8546 + const settings = yield* serverSettings.getSettings.pipe(Effect.orDie);
8547 + if (
8548 + !resolveProjectProviderInstanceEnabled(
8549 + settings,
8550 + thread.projectId,
8551 + thread.providerInstanceId,
8552 + )
8553 + ) {
8554 + return yield* new OrchestratorDispatchError({
8555 + commandId: command.commandId,
8556 + commandType: command.type,
8557 + cause: "The provider for this project is disabled.",
8558 + });
8559 + }
8560 + const questionIds = new Set(approvalTurnItem.questions.map((question) => question.id));
8561 + if (
8562 + !Object.entries(answers ?? {}).every(
8563 + ([questionId, answer]) =>
8564 + questionIds.has(questionId) || isMessageQuestionAnswer(answer),
8565 + )
8566 + ) {
8567 + return yield* new OrchestratorDispatchError({
8568 + commandId: command.commandId,
8569 + commandType: command.type,
8570 + cause: "Answer each question before sending.",
8571 + });
8572 + }
8573 + for (const question of approvalTurnItem.questions) {
8574 + const answer = answers?.[question.id];
8575 + const answered = isMessageQuestionAnswer(answer);
8576 + // Clients send "" for a skipped optional question, such as an attachment-only answer.
8577 + if (!answered && question.required === false) continue;
8578 + if (!answered) {
8579 + return yield* new OrchestratorDispatchError({
8580 + commandId: command.commandId,
8581 + commandType: command.type,
8582 + cause: "Answer each question before sending.",
8583 + });
8584 + }
8585 + const text = typeof answer === "string" ? answer : answer.join(", ");
8586 + replies.push(`${question.question}\n${text.trim()}`);
8587 + }
8588 + if (replies.length === 0) {
8589 + return yield* new OrchestratorDispatchError({
8590 + commandId: command.commandId,
8591 + commandType: command.type,
8592 + cause: "Enter an answer before sending.",
8593 + });
8594 + }
8595 + }
8596 +
8597 const now = yield* DateTime.now;
8598 const resolvedRequest = {
8599 ...runtimeRequest,
@@ -8590,10 +8667,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
8667 },
8668 });
8669 }
8593 - const resolvedNodeStatus =
8594 - command.decision === "decline" || command.decision === "cancel"
8595 - ? ("cancelled" as const)
8596 - : ("completed" as const);
8670 yield* emitEvent({
8671 type: "runtime-request.updated",
8672 threadId: command.threadId,
@@ -8629,7 +8702,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
8702 });
8703 }
8704
8632 - const approvalTurnItem = context.item;
8705 if (approvalTurnItem !== undefined) {
8706 yield* emitEvent({
8707 type: "turn-item.updated",
@@ -8668,33 +8740,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
8740 }
8741 if (runtimeRequest.responseCapability.type === "message") {
8742 if (resolvedNodeStatus === "cancelled") return;
8671 - if (approvalTurnItem?.type !== "user_input_request") {
8672 - return yield* new OrchestratorDispatchError({
8673 - commandId: command.commandId,
8674 - commandType: command.type,
8675 - cause: "The question for this request was not found.",
8676 - });
8677 - }
8678 - const replies: string[] = [];
8679 - for (const question of approvalTurnItem.questions) {
8680 - const answer = answers?.[question.id];
8681 - if (typeof answer !== "string" || answer.trim().length === 0) {
8682 - if (question.required === false) continue;
8683 - return yield* new OrchestratorDispatchError({
8684 - commandId: command.commandId,
8685 - commandType: command.type,
8686 - cause: "Answer each question before sending.",
8687 - });
8688 - }
8689 - replies.push(`${question.question}\n${answer.trim()}`);
8690 - }
8691 - if (replies.length === 0) {
8692 - return yield* new OrchestratorDispatchError({
8693 - commandId: command.commandId,
8694 - commandType: command.type,
8695 - cause: "Enter an answer before sending.",
8696 - });
8697 - }
8743 let dispatchMode: Extract<
8744 OrchestrationV2Command,
8745 { type: "message.dispatch" }
@@ -8728,21 +8773,19 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
8773 dispatchMode = { type: "steer_active", targetRunId: activeRun.id };
8774 }
8775 }
8731 - return yield* dispatchMessage(
8732 - {
8733 - type: "message.dispatch",
8734 - commandId: command.commandId,
8735 - threadId: command.threadId,
8736 - messageId: MessageId.make(`async-answer:${command.requestId}`),
8737 - text: replies.join("\n\n"),
8738 - attachments: [],
8739 - createdBy: "user",
8740 - creationSource: "server",
8741 - dispatchMode,
8742 - },
8743 - events,
8744 - effects,
8745 - );
8776 + const answer = {
8777 + type: "message.dispatch" as const,
8778 + commandId: command.commandId,
8779 + threadId: command.threadId,
8780 + messageId: MessageId.make(`async-answer:${command.requestId}`),
8781 + text: replies.join("\n\n"),
8782 + attachments: [],
8783 + createdBy: "user" as const,
8784 + creationSource: "server" as const,
8785 + dispatchMode,
8786 + };
8787 + yield* dispatchMessage(answer, events, effects);
8788 + return yield* recordSubagentFollowup(answer, events).pipe(mapDispatchError(answer));
8789 }
8790 if (providerSessionId === null) return;
8791 yield* Ref.update(effects, (existing) => [
@@ -12968,7 +13011,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
13011 commandThreadId(command),
13012 dispatchWithReceiptEffect(command),
13013 );
12971 - if (command.type !== "message.dispatch" || !mayStartSubagentFollowup(command))
13014 + // Answering a message-mode question can also start a follow-up.
13015 + if (
13016 + command.type !== "runtime-request.respond" &&
13017 + (command.type !== "message.dispatch" || !mayStartSubagentFollowup(command))
13018 + )
13019 return yield* dispatch;
13020 const parentThreadId = yield* appOwnedSubagentParentThreadId(command.threadId).pipe(
13021 Effect.mapError(
apps/server/src/orchestration-v2/ProjectionStore.ts
+11
@@ -4508,6 +4508,10 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
4508 AND node.status IN ('pending', 'starting', 'running', 'waiting')
4509 AND (
4510 (node.run_id IS NULL AND node.kind = 'root_turn')
4511 + OR node.node_id IN (
4512 + SELECT node_id FROM orchestration_v2_projection_runtime_requests
4513 + WHERE thread_id = ${threadId} AND status = 'pending' AND kind = 'user_input'
4514 + )
4515 OR node.run_id IN (
4516 SELECT run_id FROM orchestration_v2_projection_runs
4517 WHERE thread_id = ${threadId}
@@ -4686,6 +4690,13 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
4690 AND status IN ('queued', 'preparing', 'starting', 'running', 'waiting')
4691 )
4692 OR item.type IN ('command_execution', 'dynamic_tool', 'subagent')
4693 + OR (
4694 + item.type = 'user_input_request'
4695 + AND json_extract(item.payload_json, '$.requestId') IN (
4696 + SELECT runtime_request_id FROM orchestration_v2_projection_runtime_requests
4697 + WHERE thread_id = ${threadId} AND status = 'pending' AND kind = 'user_input'
4698 + )
4699 + )
4700 OR (
4701 item.run_id IS NULL
4702 AND item.node_id IN (
apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts
+36 -4
@@ -2,6 +2,7 @@ import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
2 import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
3 import {
4 CommandId,
5 + isProviderNativeSubagentThread,
6 type OrchestrationV2DomainEvent,
7 type ProviderThreadId,
8 type OrchestrationV2RestartCancelledBackgroundWork,
@@ -232,16 +233,37 @@ export const make = Effect.gen(function* () {
233 }
234 runs.push(run);
235 }
236 + const pendingRequests = projection.runtimeRequests.filter(
237 + (request) => request.status === "pending",
238 + );
239 + // The lost provider callback cannot resume; persisted questions can be
240 + // answered by a new user message in threads that accept messages.
241 + const questions = pendingRequests.filter(
242 + (request) =>
243 + request.kind === "user_input" &&
244 + request.responseCapability.type === "live" &&
245 + projection.turnItems.some(
246 + (item) =>
247 + item.type === "user_input_request" &&
248 + item.requestId === request.id &&
249 + item.nodeId === request.nodeId &&
250 + item.questions.length > 0 &&
251 + item.questions.every((question) => question.question.trim().length > 0),
252 + ) &&
253 + !isProviderNativeSubagentThread(projection.thread),
254 + );
255 + const recoveredRequestIds = new Set(questions.map((request) => request.id));
256 const messageRequestNodeIds = new Set(
236 - projection.runtimeRequests
257 + pendingRequests
258 .filter(
259 (request) =>
239 - request.status === "pending" && request.responseCapability.type === "message",
260 + request.responseCapability.type === "message" || recoveredRequestIds.has(request.id),
261 )
262 .map((request) => request.nodeId),
263 );
243 - const requests = projection.runtimeRequests.filter(
244 - (request) => request.status === "pending" && request.responseCapability.type !== "message",
264 + const requests = pendingRequests.filter(
265 + (request) =>
266 + request.responseCapability.type !== "message" && !recoveredRequestIds.has(request.id),
267 );
268 // Delegated task rows, items and nodes stay open: the child settles them.
269 const delegatedTaskNodeIds = new Set<string>([
@@ -338,6 +360,16 @@ export const make = Effect.gen(function* () {
360 payload: { ...run, queueHeld: true },
361 });
362 }
363 + for (const question of questions) {
364 + events.push({
365 + id: yield* allocateEventId(),
366 + type: "runtime-request.updated",
367 + threadId: projection.thread.id,
368 + nodeId: question.nodeId,
369 + occurredAt: now,
370 + payload: { ...question, responseCapability: { type: "message" } },
371 + });
372 + }
373 for (const request of requests) {
374 events.push({
375 id: yield* allocateEventId(),
apps/server/src/orchestration-v2/ProviderSessionManager.test.ts
+86
@@ -2900,6 +2900,92 @@ it.effect("ProviderSessionManagerV2 settles a request the event pump persists du
2900 }),
2901 );
2902
2903 +it.effect.each(["manual_shutdown", "server_shutdown"] as const)(
2904 + "ProviderSessionManagerV2 preserves questions only for server shutdown (%s)",
2905 + (reason) =>
2906 + Effect.gen(function* () {
2907 + const state = yield* Ref.make(emptyState);
2908 + const effect = Effect.gen(function* () {
2909 + const eventSink = yield* EventSink.EventSinkV2;
2910 + const idAllocator = yield* IdAllocator.IdAllocatorV2;
2911 + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
2912 + const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
2913 + const now = yield* DateTime.now;
2914 + const projectId = yield* idAllocator.allocate.project({
2915 + fixtureName: "provider-session-manager-request-expire",
2916 + });
2917 + const threadId = yield* idAllocator.allocate.thread({
2918 + fixtureName: "provider-session-manager-request-expire",
2919 + projectId,
2920 + });
2921 + const providerSessionId = yield* idAllocator.allocate.providerSession({
2922 + providerInstanceId: modelSelection.instanceId,
2923 + threadId,
2924 + });
2925 + const providerThread = makeProviderThread({
2926 + idAllocator,
2927 + threadId,
2928 + providerSessionId,
2929 + now,
2930 + });
2931 +
2932 + yield* eventSink.write({
2933 + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })],
2934 + });
2935 + const pendingRequest = yield* makePendingRuntimeRequestEvents({
2936 + idAllocator,
2937 + threadId,
2938 + providerSessionId,
2939 + providerThread,
2940 + now,
2941 + });
2942 + yield* eventSink.write({
2943 + events: pendingRequest.events.map((event) =>
2944 + event.type === "turn-item.updated"
2945 + ? {
2946 + ...event,
2947 + payload: {
2948 + ...event.payload,
2949 + type: "user_input_request",
2950 + questions: [
2951 + { id: "color", header: "Color", question: "Which color?", options: [] },
2952 + ],
2953 + },
2954 + }
2955 + : event.type === "runtime-request.updated"
2956 + ? { ...event, payload: { ...event.payload, kind: "user_input" } }
2957 + : event,
2958 + ),
2959 + });
2960 + yield* manager.open({
2961 + threadId,
2962 + providerSessionId,
2963 + modelSelection,
2964 + runtimePolicy,
2965 + });
2966 + yield* manager.release({
2967 + providerSessionId,
2968 + reason,
2969 + });
2970 +
2971 + const projection = yield* projectionStore.getThreadProjection(threadId);
2972 + const request = projection.runtimeRequests.at(-1);
2973 + const requestNode = projection.nodes.find((node) => node.id === request?.nodeId);
2974 + const requestTurnItem = projection.turnItems.find(
2975 + (item) => item.type === "user_input_request" && item.requestId === request?.id,
2976 + );
2977 +
2978 + const preserved = reason === "server_shutdown";
2979 + assert.equal(request?.status, preserved ? "pending" : "cancelled");
2980 + assert.equal(request?.responseCapability.type, preserved ? "live" : "not_resumable");
2981 + assert.equal(requestNode?.status, preserved ? "waiting" : "cancelled");
2982 + assert.equal(requestTurnItem?.status, preserved ? "waiting" : "cancelled");
2983 + });
2984 +
2985 + yield* effect.pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 1000 })));
2986 + }),
2987 +);
2988 +
2989 it.effect("ProviderSessionManagerV2 persists session-scoped runtime requests without a run", () =>
2990 Effect.gen(function* () {
2991 const state = yield* Ref.make(emptyState);
apps/server/src/orchestration-v2/ProviderSessionManager.ts
+3 -1
@@ -633,7 +633,9 @@ export const layerWithOptions = (
633 request.status === "pending" &&
634 request.responseCapability.type === "live" &&
635 request.responseCapability.providerSessionId === providerSessionId &&
636 - DateTime.isLessThanOrEqualTo(request.createdAt, input.releasedAt),
636 + DateTime.isLessThanOrEqualTo(request.createdAt, input.releasedAt) &&
637 + // Shutdown reconciliation decides which questions can survive the lost callback.
638 + !(input.reason === "server_shutdown" && request.kind === "user_input"),
639 );
640
641 for (const request of releasedRequests) {
apps/server/src/orchestration-v2/runtimeLayer.test.ts
+315 -93
@@ -18,6 +18,7 @@ import {
18 TurnItemId,
19 type ModelSelection,
20 type OrchestrationV2Run,
21 + type OrchestrationV2RuntimeRequest,
22 ProjectId,
23 type PullRequestDetail,
24 type PullRequestComment,
@@ -1077,15 +1078,288 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => {
1078 }),
1079 );
1080
1081 + const seedPendingQuestion = Effect.fnUntraced(function* (
1082 + threadId: ThreadId,
1083 + runId: RunId | null,
1084 + responseCapability: OrchestrationV2RuntimeRequest["responseCapability"],
1085 + ) {
1086 + const eventSink = yield* EventSink.EventSinkV2;
1087 + const now = yield* DateTime.now;
1088 + const requestId = RuntimeRequestId.make(`${threadId}:request`);
1089 + const nodeId = NodeId.make(`${threadId}:node`);
1090 + const itemId = TurnItemId.make(`${threadId}:item`);
1091 + yield* eventSink.write({
1092 + commandId: CommandId.make(`${threadId}:seed`),
1093 + events: [
1094 + {
1095 + id: EventId.make(`${threadId}:node-event`),
1096 + type: "node.updated",
1097 + threadId,
1098 + nodeId,
1099 + occurredAt: now,
1100 + payload: {
1101 + id: nodeId,
1102 + threadId,
1103 + runId,
1104 + parentNodeId: null,
1105 + rootNodeId: nodeId,
1106 + kind: "user_input_request",
1107 + status: "waiting",
1108 + countsForRun: false,
1109 + providerThreadId: null,
1110 + providerTurnId: null,
1111 + nativeItemRef: null,
1112 + runtimeRequestId: requestId,
1113 + checkpointScopeId: null,
1114 + startedAt: now,
1115 + completedAt: null,
1116 + },
1117 + },
1118 + {
1119 + id: EventId.make(`${threadId}:request-event`),
1120 + type: "runtime-request.updated",
1121 + threadId,
1122 + nodeId,
1123 + occurredAt: now,
1124 + payload: {
1125 + id: requestId,
1126 + nodeId,
1127 + providerTurnId: null,
1128 + nativeRequestRef: null,
1129 + kind: "user_input",
1130 + status: "pending",
1131 + responseCapability,
1132 + createdAt: now,
1133 + resolvedAt: null,
1134 + },
1135 + },
1136 + {
1137 + id: EventId.make(`${threadId}:item-event`),
1138 + type: "turn-item.updated",
1139 + threadId,
1140 + nodeId,
1141 + occurredAt: now,
1142 + payload: {
1143 + id: itemId,
1144 + type: "user_input_request",
1145 + threadId,
1146 + runId,
1147 + nodeId,
1148 + providerThreadId: null,
1149 + providerTurnId: null,
1150 + nativeItemRef: null,
1151 + parentItemId: null,
1152 + ordinal: 0,
1153 + status: "waiting",
1154 + title: null,
1155 + startedAt: now,
1156 + completedAt: null,
1157 + updatedAt: now,
1158 + requestId,
1159 + questions: [
1160 + { id: "color", header: "Color", question: "Which color?", options: [] },
1161 + {
1162 + id: "sizes",
1163 + header: "Sizes",
1164 + question: "Which sizes?",
1165 + options: [],
1166 + multiSelect: true,
1167 + },
1168 + {
1169 + id: "note",
1170 + header: "Note",
1171 + question: "Anything else?",
1172 + options: [],
1173 + required: false,
1174 + },
1175 + ],
1176 + },
1177 + },
1178 + ],
1179 + });
1180 + return { requestId, nodeId, itemId };
1181 + });
1182 +
1183 + it.effect.each([
1184 + { trigger: "startup", scope: "run" },
1185 + { trigger: "shutdown", scope: "session" },
1186 + ] as const)(
1187 + "recovers only reconstructable user questions and keeps their cards pending on $trigger ($scope)",
1188 + ({ trigger, scope }) =>
1189 + Effect.gen(function* () {
1190 + const orchestrator = yield* Orchestrator.OrchestratorV2;
1191 + const eventSink = yield* EventSink.EventSinkV2;
1192 + const now = yield* DateTime.now;
1193 + const threadId = ThreadId.make(`runtime-recovered-question-${trigger}-${scope}`);
1194 + yield* orchestrator.dispatch({
1195 + type: "thread.create",
1196 + commandId: CommandId.make(`runtime-recovered-question-create-${trigger}-${scope}`),
1197 + createdBy: "user",
1198 + creationSource: "web",
1199 + threadId,
1200 + projectId: ProjectId.make(`runtime-recovered-question-project-${trigger}-${scope}`),
1201 + title: "Async question",
1202 + modelSelection,
1203 + runtimeMode: "full-access",
1204 + interactionMode: "default",
1205 + branch: null,
1206 + worktreePath: process.cwd(),
1207 + });
1208 + if (scope === "run") {
1209 + yield* orchestrator.dispatch({
1210 + type: "message.dispatch",
1211 + commandId: CommandId.make(`${threadId}:run`),
1212 + threadId,
1213 + messageId: MessageId.make(`${threadId}:message`),
1214 + text: "Ask me a question",
1215 + attachments: [],
1216 + createdBy: "user",
1217 + creationSource: "web",
1218 + dispatchMode: { type: "start_immediately" },
1219 + });
1220 + }
1221 + const runId = (yield* orchestrator.getThreadProjection(threadId)).runs[0]?.id ?? null;
1222 + const { requestId, nodeId, itemId } = yield* seedPendingQuestion(threadId, runId, {
1223 + type: "live",
1224 + providerSessionId: ProviderSessionId.make(`lost-session-${trigger}-${scope}`),
1225 + });
1226 + const recovery = yield* ProviderRuntimeRecoveryService.ProviderRuntimeRecoveryService;
1227 + const original = yield* orchestrator.getThreadProjection(threadId);
1228 + const request = original.runtimeRequests[0]!;
1229 + const questionItem = original.turnItems.find((item) => item.type === "user_input_request")!;
1230 + const nativeThreadId = ThreadId.make(`runtime-recovered-native-child-${trigger}-${scope}`);
1231 + const nativeRequestId = RuntimeRequestId.make(
1232 + `runtime-recovered-native-request-${trigger}-${scope}`,
1233 + );
1234 + const approvalId = RuntimeRequestId.make(`runtime-recovered-approval-${trigger}-${scope}`);
1235 + const missingTextId = RuntimeRequestId.make(
1236 + `runtime-recovered-missing-text-${trigger}-${scope}`,
1237 + );
1238 + yield* eventSink.write({
1239 + events: [
1240 + {
1241 + id: EventId.make(`recovery-approval-${trigger}-${scope}`),
1242 + type: "runtime-request.updated",
1243 + threadId,
1244 + occurredAt: now,
1245 + payload: { ...request, id: approvalId, kind: "command" },
1246 + },
1247 + {
1248 + id: EventId.make(`recovery-missing-text-${trigger}-${scope}`),
1249 + type: "runtime-request.updated",
1250 + threadId,
1251 + occurredAt: now,
1252 + payload: { ...request, id: missingTextId },
1253 + },
1254 + {
1255 + id: EventId.make(`recovery-native-thread-${trigger}-${scope}`),
1256 + type: "thread.created",
1257 + threadId: nativeThreadId,
1258 + occurredAt: now,
1259 + payload: {
1260 + ...original.thread,
1261 + id: nativeThreadId,
1262 + creationSource: "provider",
1263 + lineage: {
1264 + parentThreadId: threadId,
1265 + rootThreadId: threadId,
1266 + relationshipToParent: "subagent",
1267 + },
1268 + },
1269 + },
1270 + {
1271 + id: EventId.make(`recovery-native-request-${trigger}-${scope}`),
1272 + type: "runtime-request.updated",
1273 + threadId: nativeThreadId,
1274 + occurredAt: now,
1275 + payload: { ...request, id: nativeRequestId },
1276 + },
1277 + {
1278 + id: EventId.make(`recovery-native-item-${trigger}-${scope}`),
1279 + type: "turn-item.updated",
1280 + threadId: nativeThreadId,
1281 + occurredAt: now,
1282 + payload: {
1283 + ...questionItem,
1284 + id: TurnItemId.make(`recovery-native-item-${trigger}-${scope}`),
1285 + type: "user_input_request",
1286 + threadId: nativeThreadId,
1287 + runId: null,
1288 + requestId: nativeRequestId,
1289 + questions: [
1290 + { id: "color", header: "Color", question: "Which color?", options: [] },
1291 + ],
1292 + },
1293 + },
1294 + ],
1295 + });
1296 + yield* recovery.reconcile(trigger);
1297 + const recovered = yield* orchestrator.getThreadProjection(threadId);
1298 + if (scope === "run") {
1299 + assert.equal(recovered.runs[0]?.status, "cancelled");
1300 + } else {
1301 + assert.isEmpty(recovered.runs);
1302 + }
1303 + assert.deepEqual(
1304 + recovered.runtimeRequests.find((row) => row.id === requestId),
1305 + {
1306 + ...request,
1307 + responseCapability: { type: "message" },
1308 + },
1309 + );
1310 + assert.equal(recovered.nodes.find((row) => row.id === nodeId)?.status, "waiting");
1311 + assert.equal(recovered.turnItems.find((row) => row.id === itemId)?.status, "waiting");
1312 + for (const id of [approvalId, missingTextId]) {
1313 + const closed = recovered.runtimeRequests.find((row) => row.id === id);
1314 + assert.equal(closed?.status, trigger === "startup" ? "expired" : "cancelled");
1315 + assert.equal(closed?.responseCapability.type, "not_resumable");
1316 + }
1317 + const native = yield* orchestrator.getThreadProjection(nativeThreadId);
1318 + assert.equal(
1319 + native.runtimeRequests[0]?.status,
1320 + trigger === "startup" ? "expired" : "cancelled",
1321 + );
1322 + assert.equal(native.runtimeRequests[0]?.responseCapability.type, "not_resumable");
1323 + yield* recovery.reconcile("shutdown");
1324 + yield* recovery.reconcile("startup");
1325 + assert.deepEqual(
1326 + (yield* orchestrator.getThreadProjection(threadId)).runtimeRequests,
1327 + recovered.runtimeRequests,
1328 + );
1329 + yield* orchestrator.dispatch({
1330 + type: "runtime-request.respond",
1331 + commandId: CommandId.make(`recovery-answer-${trigger}-${scope}`),
1332 + threadId,
1333 + requestId,
1334 + answers: { color: "Blue", sizes: ["Small", "Large"] },
1335 + });
1336 + const answered = yield* orchestrator.getThreadProjection(threadId);
1337 + assert.equal(
1338 + answered.runtimeRequests.find((row) => row.id === requestId)?.status,
1339 + "resolved",
1340 + );
1341 + assert.equal(answered.nodes.find((row) => row.id === nodeId)?.status, "completed");
1342 + assert.equal(answered.turnItems.find((row) => row.id === itemId)?.status, "completed");
1343 + assert.lengthOf(answered.runs, recovered.runs.length + 1);
1344 + assert.equal(answered.messages.at(-1)?.runId, answered.runs.at(-1)?.id);
1345 + assert.equal(
1346 + answered.messages.at(-1)?.text,
1347 + "Which color?\nBlue\n\nWhich sizes?\nSmall, Large",
1348 + );
1349 + assert.deepEqual(
1350 + (yield* (yield* EffectOutbox.EffectOutboxV2).listByCommandId(
1351 + CommandId.make(`recovery-answer-${trigger}-${scope}`),
1352 + )).map((effect) => effect.request.type),
1353 + ["provider-turn.start"],
1354 + );
1355 + }),
1356 + );
1357 +
1358 it.effect("answers an async question after its provider exits and commits the answer once", () =>
1359 Effect.gen(function* () {
1360 const orchestrator = yield* Orchestrator.OrchestratorV2;
1361 const eventSink = yield* EventSink.EventSinkV2;
1084 - const now = yield* DateTime.now;
1362 const threadId = ThreadId.make("runtime-async-question");
1086 - const requestId = RuntimeRequestId.make("runtime-async-question-request");
1087 - const nodeId = NodeId.make("runtime-async-question-node");
1088 - const itemId = TurnItemId.make("runtime-async-question-item");
1363 yield* orchestrator.dispatch({
1364 type: "thread.create",
1365 commandId: CommandId.make("runtime-async-question-create"),
@@ -1100,100 +1374,40 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => {
1374 branch: null,
1375 worktreePath: process.cwd(),
1376 });
1103 - yield* eventSink.write({
1104 - commandId: CommandId.make("runtime-async-question-seed"),
1105 - events: [
1106 - {
1107 - id: EventId.make("runtime-async-question-node-event"),
1108 - type: "node.updated",
1109 - threadId,
1110 - nodeId,
1111 - occurredAt: now,
1112 - payload: {
1113 - id: nodeId,
1114 - threadId,
1115 - runId: null,
1116 - parentNodeId: null,
1117 - rootNodeId: nodeId,
1118 - kind: "user_input_request",
1119 - status: "waiting",
1120 - countsForRun: false,
1121 - providerThreadId: null,
1122 - providerTurnId: null,
1123 - nativeItemRef: null,
1124 - runtimeRequestId: requestId,
1125 - checkpointScopeId: null,
1126 - startedAt: now,
1127 - completedAt: null,
1128 - },
1129 - },
1130 - {
1131 - id: EventId.make("runtime-async-question-request-event"),
1132 - type: "runtime-request.updated",
1133 - threadId,
1134 - nodeId,
1135 - occurredAt: now,
1136 - payload: {
1137 - id: requestId,
1138 - nodeId,
1139 - providerTurnId: null,
1140 - nativeRequestRef: null,
1141 - kind: "user_input",
1142 - status: "pending",
1143 - responseCapability: { type: "message" },
1144 - createdAt: now,
1145 - resolvedAt: null,
1146 - },
1147 - },
1148 - {
1149 - id: EventId.make("runtime-async-question-item-event"),
1150 - type: "turn-item.updated",
1151 - threadId,
1152 - nodeId,
1153 - occurredAt: now,
1154 - payload: {
1155 - id: itemId,
1156 - type: "user_input_request",
1157 - threadId,
1158 - runId: null,
1159 - nodeId,
1160 - providerThreadId: null,
1161 - providerTurnId: null,
1162 - nativeItemRef: null,
1163 - parentItemId: null,
1164 - ordinal: 0,
1165 - status: "waiting",
1166 - title: null,
1167 - startedAt: now,
1168 - completedAt: null,
1169 - updatedAt: now,
1170 - requestId,
1171 - responseMode: "message",
1172 - questions: [{ id: "color", header: "Color", question: "Which color?", options: [] }],
1173 - },
1174 - },
1175 - ],
1377 + const { requestId, nodeId, itemId } = yield* seedPendingQuestion(threadId, null, {
1378 + type: "message",
1379 });
1177 - const invalid = yield* orchestrator
1178 - .dispatch({
1179 - type: "runtime-request.respond",
1180 - commandId: CommandId.make("runtime-async-question-blank"),
1181 - threadId,
1182 - requestId,
1183 - answers: { color: " " },
1184 - })
1185 - .pipe(Effect.result);
1186 - assert.equal(invalid._tag, "Failure");
1187 - const unanswered = yield* orchestrator.getThreadProjection(threadId);
1188 - assert.equal(unanswered.runtimeRequests[0]?.status, "pending");
1189 - assert.deepEqual(unanswered.messages, []);
1380 + const outbox = yield* EffectOutbox.EffectOutboxV2;
1381 + const beforeInvalid = yield* orchestrator.getThreadProjection(threadId);
1382 + const beforeSequence = yield* eventSink.latestSequence();
1383 + for (const [index, answers] of [
1384 + { color: " ", sizes: ["Small"] },
1385 + { color: "Blue", sizes: [] },
1386 + { color: "Blue", sizes: ["Small", " "] },
1387 + { color: "Blue", sizes: ["Small"], extra: " " },
1388 + ].entries()) {
1389 + const commandId = CommandId.make(`runtime-async-question-invalid:${index}`);
1390 + const invalid = yield* orchestrator
1391 + .dispatch({
1392 + type: "runtime-request.respond",
1393 + commandId,
1394 + threadId,
1395 + requestId,
1396 + answers,
1397 + })
1398 + .pipe(Effect.result);
1399 + assert.equal(invalid._tag, "Failure", JSON.stringify(answers));
1400 + assert.deepEqual(yield* orchestrator.getThreadProjection(threadId), beforeInvalid);
1401 + assert.equal(yield* eventSink.latestSequence(), beforeSequence);
1402 + assert.deepEqual(yield* outbox.listByCommandId(commandId), []);
1403 + }
1404
1405 const command = {
1406 type: "runtime-request.respond" as const,
1407 commandId: CommandId.make("runtime-async-question-answer"),
1408 threadId,
1409 requestId,
1196 - answers: { color: " Blue " },
1410 + answers: { color: " Blue ", sizes: ["Small", "Large"], note: "" },
1411 };
1412 const accepted = yield* orchestrator.dispatch(command);
1413 const repeated = yield* orchestrator.dispatch(command);
@@ -1210,13 +1424,21 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => {
1424 requestId,
1425 answers: command.answers,
1426 attachmentsByQuestionId: {},
1213 - questionTextById: { color: "Which color?" },
1427 + questionTextById: {
1428 + color: "Which color?",
1429 + sizes: "Which sizes?",
1430 + note: "Anything else?",
1431 + },
1432 });
1433 }
1434 assert.equal(answered.messages.length, 1);
1217 - assert.equal(answered.messages[0]?.text, "Which color?\nBlue");
1435 + assert.equal(answered.messages[0]?.text, "Which color?\nBlue\n\nWhich sizes?\nSmall, Large");
1436 assert.equal(answered.messages[0]?.role, "user");
1437 assert.equal(answered.runs.length, 1);
1438 + assert.deepEqual(
1439 + (yield* outbox.listByCommandId(command.commandId)).map((effect) => effect.request.type),
1440 + ["provider-turn.start"],
1441 + );
1442
1443 const duplicate = yield* orchestrator
1444 .dispatch({