The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 14:30 UTC
fix(server): agent questions stay answerable after a server restart (#19229)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Julius Marminge <julius0216@outlook.com>
Dara Adedeji committed
Oct 27, 2026 at 19:11 UTC
867112cda0906960f6340171c5424600a1ac1d28
7 files changed
+718
-147
apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts
+172
-1
@@ -15,11 +15,14 @@ import {
15
ProviderInstanceId,
16
ProviderThreadId,
17
RunId,
18
+ RuntimeRequestId,
19
ThreadId,
20
TurnItemId,
21
} from "@t3tools/contracts";
22
import * as DateTime from "effect/DateTime";
23
import * as Effect from "effect/Effect";
24
+import * as Deferred from "effect/Deferred";
25
+import * as Fiber from "effect/Fiber";
26
import * as Layer from "effect/Layer";
27
import * as Stream from "effect/Stream";
28
@@ -40,6 +43,7 @@ import * as EventSink from "./EventSink.ts";
43
import * as Orchestrator from "./Orchestrator.ts";
44
import { continueRestartedRun } from "./RestartContinuation.ts";
45
import * as RuntimeLayer from "./runtimeLayer.ts";
46
+import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
47
import * as ProviderTurnStartServiceTestkit from "./ProviderTurnStartService.testkit.ts";
48
import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
49
import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests";
@@ -104,7 +108,11 @@ const layerTestProviderInstanceRegistry = Layer.succeed(
108
},
109
);
110
107
-const layerTest = Layer.mergeAll(RuntimeLayer.layer, RuntimeLayer.layerEventSink).pipe(
111
+const layerTest = Layer.mergeAll(
112
+ RuntimeLayer.layer,
113
+ RuntimeLayer.layerEventSink,
114
+ ThreadCommandExecutor.layer,
115
+).pipe(
116
Layer.provideMerge(RuntimeLayer.layerProjectService),
117
Layer.provide(
118
Layer.mock(WorkspacePaths.WorkspacePaths)({
@@ -1980,6 +1988,169 @@ it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))(
1988
}
1989
}),
1990
);
1991
+
1992
+ it.effect("serializes a recovered child answer with its parent and tracks the follow-up", () =>
1993
+ Effect.gen(function* () {
1994
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
1995
+ const eventSink = yield* EventSink.EventSinkV2;
1996
+ const now = yield* DateTime.now;
1997
+ const parentThreadId = ThreadId.make("thread:question-answer-parent");
1998
+ const projectId = ProjectId.make("project:question-answer");
1999
+ const parentRunId = RunId.make("run:question-answer-parent");
2000
+ const rootNodeId = NodeId.make("node:question-answer-parent");
2001
+ yield* seedParentWithTerminalTask({
2002
+ threadId: parentThreadId,
2003
+ projectId,
2004
+ runId: parentRunId,
2005
+ rootNodeId,
2006
+ taskId: NodeId.make("node:question-answer-seed"),
2007
+ deliveryState: "acknowledged",
2008
+ now,
2009
+ });
2010
+ const child = yield* seedRestartCancelledChild({
2011
+ parentThreadId,
2012
+ projectId,
2013
+ parentRunId,
2014
+ rootNodeId,
2015
+ name: "question-answer-child",
2016
+ completionWake: "always",
2017
+ continuationPending: false,
2018
+ now,
2019
+ });
2020
+ // The cancelled result is published before anyone answers the question.
2021
+ yield* orchestrator.recoverDelegatedTasks;
2022
+ assert.isNotNull(
2023
+ (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find(
2024
+ (task) => task.id === child.taskId,
2025
+ )?.result,
2026
+ );
2027
+ const parentRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs.find(
2028
+ (run) => run.id === parentRunId,
2029
+ )!;
2030
+ yield* eventSink.write({
2031
+ events: [
2032
+ {
2033
+ id: EventId.make("event:question-answer-parent-completed"),
2034
+ type: "run.updated",
2035
+ threadId: parentThreadId,
2036
+ runId: parentRunId,
2037
+ occurredAt: now,
2038
+ payload: { ...parentRun, status: "completed", completedAt: now },
2039
+ },
2040
+ ],
2041
+ });
2042
+ const requestId = RuntimeRequestId.make("request:question-answer");
2043
+ const nodeId = NodeId.make("node:question-answer-request");
2044
+ yield* eventSink.write({
2045
+ commandId: CommandId.make("command:seed-question-answer"),
2046
+ events: [
2047
+ {
2048
+ id: EventId.make("event:question-answer-request"),
2049
+ type: "runtime-request.updated",
2050
+ threadId: child.childThreadId,
2051
+ nodeId,
2052
+ occurredAt: now,
2053
+ payload: {
2054
+ id: requestId,
2055
+ nodeId,
2056
+ providerTurnId: null,
2057
+ nativeRequestRef: null,
2058
+ kind: "user_input",
2059
+ status: "pending",
2060
+ responseCapability: { type: "message" },
2061
+ createdAt: now,
2062
+ resolvedAt: null,
2063
+ },
2064
+ },
2065
+ {
2066
+ id: EventId.make("event:question-answer-item"),
2067
+ type: "turn-item.updated",
2068
+ threadId: child.childThreadId,
2069
+ nodeId,
2070
+ occurredAt: now,
2071
+ payload: {
2072
+ id: TurnItemId.make("turn-item:question-answer"),
2073
+ type: "user_input_request",
2074
+ threadId: child.childThreadId,
2075
+ runId: child.childRunId,
2076
+ nodeId,
2077
+ providerThreadId: null,
2078
+ providerTurnId: null,
2079
+ nativeItemRef: null,
2080
+ parentItemId: null,
2081
+ ordinal: 0,
2082
+ status: "waiting",
2083
+ title: null,
2084
+ startedAt: now,
2085
+ completedAt: null,
2086
+ updatedAt: now,
2087
+ requestId,
2088
+ questions: [
2089
+ { id: "color", header: "Color", question: "Which color?", options: [] },
2090
+ ],
2091
+ },
2092
+ },
2093
+ ],
2094
+ });
2095
+
2096
+ const executor = yield* ThreadCommandExecutor.ThreadCommandExecutor;
2097
+ const parentHeld = yield* Deferred.make<void>();
2098
+ const releaseParent = yield* Deferred.make<void>();
2099
+ const answerWaiting = yield* Deferred.make<void>();
2100
+ const withLock = executor.withLock;
2101
+ const parentMutation = yield* withLock(
2102
+ parentThreadId,
2103
+ Deferred.succeed(parentHeld, undefined).pipe(
2104
+ Effect.andThen(Deferred.await(releaseParent)),
2105
+ ),
2106
+ ).pipe(Effect.forkScoped({ startImmediately: true }));
2107
+ yield* Deferred.await(parentHeld);
2108
+ const observeLock: ThreadCommandExecutor.ThreadCommandExecutor["Service"]["withLock"] = (
2109
+ key,
2110
+ effect,
2111
+ ) =>
2112
+ key === parentThreadId
2113
+ ? Deferred.succeed(answerWaiting, undefined).pipe(Effect.andThen(withLock(key, effect)))
2114
+ : withLock(key, effect);
2115
+ const spy = vi.spyOn(executor, "withLock").mockImplementation(observeLock);
2116
+ yield* Effect.gen(function* () {
2117
+ const answer = yield* orchestrator
2118
+ .dispatch({
2119
+ type: "runtime-request.respond",
2120
+ commandId: CommandId.make("command:question-answer"),
2121
+ threadId: child.childThreadId,
2122
+ requestId,
2123
+ answers: { color: "Blue" },
2124
+ })
2125
+ .pipe(Effect.forkScoped({ startImmediately: true }));
2126
+ yield* Effect.raceFirst(Deferred.await(answerWaiting), Fiber.join(answer));
2127
+ const waiting = yield* orchestrator.getThreadProjection(child.childThreadId);
2128
+ assert.equal(
2129
+ waiting.runtimeRequests.find((row) => row.id === requestId)?.status,
2130
+ "pending",
2131
+ );
2132
+ assert.isFalse((yield* executor.activeKeys).includes(child.childThreadId));
2133
+ yield* Deferred.succeed(releaseParent, undefined);
2134
+ yield* Fiber.join(parentMutation);
2135
+ yield* Fiber.join(answer);
2136
+ }).pipe(Effect.ensuring(Effect.sync(() => spy.mockRestore())));
2137
+
2138
+ const answerRun = (yield* orchestrator.getThreadProjection(child.childThreadId)).runs.find(
2139
+ (run) => run.userMessageId === MessageId.make(`async-answer:${requestId}`),
2140
+ );
2141
+ assert.isDefined(answerRun?.delegatedTaskId);
2142
+ const followup = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find(
2143
+ (task) => task.id === answerRun?.delegatedTaskId,
2144
+ );
2145
+ assert.equal(followup?.childThreadId, child.childThreadId);
2146
+ assert.isNull(followup?.result);
2147
+ assert.isTrue(
2148
+ (yield* orchestrator.getThreadShell(parentThreadId))?.pendingBackgroundTasks?.some(
2149
+ (task) => task.kind === "subagent" && task.childThreadId === child.childThreadId,
2150
+ ),
2151
+ );
2152
+ }),
2153
+ );
2154
},
2155
);
2156
apps/server/src/orchestration-v2/Orchestrator.ts
+95
-48
@@ -62,12 +62,14 @@ import {
62
ProviderInstanceId,
63
ProviderInteractionMode,
64
resolveProviderInstanceEnabled,
65
+ resolveProjectProviderInstanceEnabled,
66
type ProviderSessionId,
67
RunId,
68
RuntimeMode,
69
ThreadLinkedPullRequest,
70
ThreadId,
71
TurnItemId,
72
+ TrimmedNonEmptyString,
73
} from "@t3tools/contracts";
74
import { modelSelectionsEqual } from "@t3tools/shared/model";
75
import { recordThreadOpen } from "@t3tools/shared/paletteSearch";
@@ -167,6 +169,10 @@ import { planThreadDeletion } from "./ThreadDeletion.ts";
169
import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
170
import * as ServerSettings from "../serverSettings.ts";
171
172
+const isMessageQuestionAnswer = Schema.is(
173
+ Schema.Union([TrimmedNonEmptyString, Schema.NonEmptyArray(TrimmedNonEmptyString)]),
174
+);
175
+
176
export class OrchestratorDispatchError extends Schema.TaggedError<OrchestratorDispatchError>()(
177
"OrchestratorDispatchError",
178
{
@@ -8517,6 +8523,77 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
8523
);
8524
}
8525
8526
+ const approvalTurnItem = context.item;
8527
+ const resolvedNodeStatus =
8528
+ command.decision === "decline" || command.decision === "cancel"
8529
+ ? ("cancelled" as const)
8530
+ : ("completed" as const);
8531
+ const replies: string[] = [];
8532
+ if (
8533
+ runtimeRequest.responseCapability.type === "message" &&
8534
+ resolvedNodeStatus !== "cancelled"
8535
+ ) {
8536
+ if (approvalTurnItem?.type !== "user_input_request") {
8537
+ return yield* new OrchestratorDispatchError({
8538
+ commandId: command.commandId,
8539
+ commandType: command.type,
8540
+ cause: "The question for this request was not found.",
8541
+ });
8542
+ }
8543
+ const thread = yield* projectionStore
8544
+ .getThread(command.threadId)
8545
+ .pipe(mapDispatchError(command));
8546
+ const settings = yield* serverSettings.getSettings.pipe(Effect.orDie);
8547
+ if (
8548
+ !resolveProjectProviderInstanceEnabled(
8549
+ settings,
8550
+ thread.projectId,
8551
+ thread.providerInstanceId,
8552
+ )
8553
+ ) {
8554
+ return yield* new OrchestratorDispatchError({
8555
+ commandId: command.commandId,
8556
+ commandType: command.type,
8557
+ cause: "The provider for this project is disabled.",
8558
+ });
8559
+ }
8560
+ const questionIds = new Set(approvalTurnItem.questions.map((question) => question.id));
8561
+ if (
8562
+ !Object.entries(answers ?? {}).every(
8563
+ ([questionId, answer]) =>
8564
+ questionIds.has(questionId) || isMessageQuestionAnswer(answer),
8565
+ )
8566
+ ) {
8567
+ return yield* new OrchestratorDispatchError({
8568
+ commandId: command.commandId,
8569
+ commandType: command.type,
8570
+ cause: "Answer each question before sending.",
8571
+ });
8572
+ }
8573
+ for (const question of approvalTurnItem.questions) {
8574
+ const answer = answers?.[question.id];
8575
+ const answered = isMessageQuestionAnswer(answer);
8576
+ // Clients send "" for a skipped optional question, such as an attachment-only answer.
8577
+ if (!answered && question.required === false) continue;
8578
+ if (!answered) {
8579
+ return yield* new OrchestratorDispatchError({
8580
+ commandId: command.commandId,
8581
+ commandType: command.type,
8582
+ cause: "Answer each question before sending.",
8583
+ });
8584
+ }
8585
+ const text = typeof answer === "string" ? answer : answer.join(", ");
8586
+ replies.push(`${question.question}\n${text.trim()}`);
8587
+ }
8588
+ if (replies.length === 0) {
8589
+ return yield* new OrchestratorDispatchError({
8590
+ commandId: command.commandId,
8591
+ commandType: command.type,
8592
+ cause: "Enter an answer before sending.",
8593
+ });
8594
+ }
8595
+ }
8596
+
8597
const now = yield* DateTime.now;
8598
const resolvedRequest = {
8599
...runtimeRequest,
@@ -8590,10 +8667,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
8667
},
8668
});
8669
}
8593
- const resolvedNodeStatus =
8594
- command.decision === "decline" || command.decision === "cancel"
8595
- ? ("cancelled" as const)
8596
- : ("completed" as const);
8670
yield* emitEvent({
8671
type: "runtime-request.updated",
8672
threadId: command.threadId,
@@ -8629,7 +8702,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
8702
});
8703
}
8704
8632
- const approvalTurnItem = context.item;
8705
if (approvalTurnItem !== undefined) {
8706
yield* emitEvent({
8707
type: "turn-item.updated",
@@ -8668,33 +8740,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
8740
}
8741
if (runtimeRequest.responseCapability.type === "message") {
8742
if (resolvedNodeStatus === "cancelled") return;
8671
- if (approvalTurnItem?.type !== "user_input_request") {
8672
- return yield* new OrchestratorDispatchError({
8673
- commandId: command.commandId,
8674
- commandType: command.type,
8675
- cause: "The question for this request was not found.",
8676
- });
8677
- }
8678
- const replies: string[] = [];
8679
- for (const question of approvalTurnItem.questions) {
8680
- const answer = answers?.[question.id];
8681
- if (typeof answer !== "string" || answer.trim().length === 0) {
8682
- if (question.required === false) continue;
8683
- return yield* new OrchestratorDispatchError({
8684
- commandId: command.commandId,
8685
- commandType: command.type,
8686
- cause: "Answer each question before sending.",
8687
- });
8688
- }
8689
- replies.push(`${question.question}\n${answer.trim()}`);
8690
- }
8691
- if (replies.length === 0) {
8692
- return yield* new OrchestratorDispatchError({
8693
- commandId: command.commandId,
8694
- commandType: command.type,
8695
- cause: "Enter an answer before sending.",
8696
- });
8697
- }
8743
let dispatchMode: Extract<
8744
OrchestrationV2Command,
8745
{ type: "message.dispatch" }
@@ -8728,21 +8773,19 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
8773
dispatchMode = { type: "steer_active", targetRunId: activeRun.id };
8774
}
8775
}
8731
- return yield* dispatchMessage(
8732
- {
8733
- type: "message.dispatch",
8734
- commandId: command.commandId,
8735
- threadId: command.threadId,
8736
- messageId: MessageId.make(`async-answer:${command.requestId}`),
8737
- text: replies.join("\n\n"),
8738
- attachments: [],
8739
- createdBy: "user",
8740
- creationSource: "server",
8741
- dispatchMode,
8742
- },
8743
- events,
8744
- effects,
8745
- );
8776
+ const answer = {
8777
+ type: "message.dispatch" as const,
8778
+ commandId: command.commandId,
8779
+ threadId: command.threadId,
8780
+ messageId: MessageId.make(`async-answer:${command.requestId}`),
8781
+ text: replies.join("\n\n"),
8782
+ attachments: [],
8783
+ createdBy: "user" as const,
8784
+ creationSource: "server" as const,
8785
+ dispatchMode,
8786
+ };
8787
+ yield* dispatchMessage(answer, events, effects);
8788
+ return yield* recordSubagentFollowup(answer, events).pipe(mapDispatchError(answer));
8789
}
8790
if (providerSessionId === null) return;
8791
yield* Ref.update(effects, (existing) => [
@@ -12968,7 +13011,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
13011
commandThreadId(command),
13012
dispatchWithReceiptEffect(command),
13013
);
12971
- if (command.type !== "message.dispatch" || !mayStartSubagentFollowup(command))
13014
+ // Answering a message-mode question can also start a follow-up.
13015
+ if (
13016
+ command.type !== "runtime-request.respond" &&
13017
+ (command.type !== "message.dispatch" || !mayStartSubagentFollowup(command))
13018
+ )
13019
return yield* dispatch;
13020
const parentThreadId = yield* appOwnedSubagentParentThreadId(command.threadId).pipe(
13021
Effect.mapError(
apps/server/src/orchestration-v2/ProjectionStore.ts
+11
@@ -4508,6 +4508,10 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
4508
AND node.status IN ('pending', 'starting', 'running', 'waiting')
4509
AND (
4510
(node.run_id IS NULL AND node.kind = 'root_turn')
4511
+ OR node.node_id IN (
4512
+ SELECT node_id FROM orchestration_v2_projection_runtime_requests
4513
+ WHERE thread_id = ${threadId} AND status = 'pending' AND kind = 'user_input'
4514
+ )
4515
OR node.run_id IN (
4516
SELECT run_id FROM orchestration_v2_projection_runs
4517
WHERE thread_id = ${threadId}
@@ -4686,6 +4690,13 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
4690
AND status IN ('queued', 'preparing', 'starting', 'running', 'waiting')
4691
)
4692
OR item.type IN ('command_execution', 'dynamic_tool', 'subagent')
4693
+ OR (
4694
+ item.type = 'user_input_request'
4695
+ AND json_extract(item.payload_json, '$.requestId') IN (
4696
+ SELECT runtime_request_id FROM orchestration_v2_projection_runtime_requests
4697
+ WHERE thread_id = ${threadId} AND status = 'pending' AND kind = 'user_input'
4698
+ )
4699
+ )
4700
OR (
4701
item.run_id IS NULL
4702
AND item.node_id IN (
apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts
+36
-4
@@ -2,6 +2,7 @@ import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
2
import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
3
import {
4
CommandId,
5
+ isProviderNativeSubagentThread,
6
type OrchestrationV2DomainEvent,
7
type ProviderThreadId,
8
type OrchestrationV2RestartCancelledBackgroundWork,
@@ -232,16 +233,37 @@ export const make = Effect.gen(function* () {
233
}
234
runs.push(run);
235
}
236
+ const pendingRequests = projection.runtimeRequests.filter(
237
+ (request) => request.status === "pending",
238
+ );
239
+ // The lost provider callback cannot resume; persisted questions can be
240
+ // answered by a new user message in threads that accept messages.
241
+ const questions = pendingRequests.filter(
242
+ (request) =>
243
+ request.kind === "user_input" &&
244
+ request.responseCapability.type === "live" &&
245
+ projection.turnItems.some(
246
+ (item) =>
247
+ item.type === "user_input_request" &&
248
+ item.requestId === request.id &&
249
+ item.nodeId === request.nodeId &&
250
+ item.questions.length > 0 &&
251
+ item.questions.every((question) => question.question.trim().length > 0),
252
+ ) &&
253
+ !isProviderNativeSubagentThread(projection.thread),
254
+ );
255
+ const recoveredRequestIds = new Set(questions.map((request) => request.id));
256
const messageRequestNodeIds = new Set(
236
- projection.runtimeRequests
257
+ pendingRequests
258
.filter(
259
(request) =>
239
- request.status === "pending" && request.responseCapability.type === "message",
260
+ request.responseCapability.type === "message" || recoveredRequestIds.has(request.id),
261
)
262
.map((request) => request.nodeId),
263
);
243
- const requests = projection.runtimeRequests.filter(
244
- (request) => request.status === "pending" && request.responseCapability.type !== "message",
264
+ const requests = pendingRequests.filter(
265
+ (request) =>
266
+ request.responseCapability.type !== "message" && !recoveredRequestIds.has(request.id),
267
);
268
// Delegated task rows, items and nodes stay open: the child settles them.
269
const delegatedTaskNodeIds = new Set<string>([
@@ -338,6 +360,16 @@ export const make = Effect.gen(function* () {
360
payload: { ...run, queueHeld: true },
361
});
362
}
363
+ for (const question of questions) {
364
+ events.push({
365
+ id: yield* allocateEventId(),
366
+ type: "runtime-request.updated",
367
+ threadId: projection.thread.id,
368
+ nodeId: question.nodeId,
369
+ occurredAt: now,
370
+ payload: { ...question, responseCapability: { type: "message" } },
371
+ });
372
+ }
373
for (const request of requests) {
374
events.push({
375
id: yield* allocateEventId(),
apps/server/src/orchestration-v2/ProviderSessionManager.test.ts
+86
@@ -2900,6 +2900,92 @@ it.effect("ProviderSessionManagerV2 settles a request the event pump persists du
2900
}),
2901
);
2902
2903
+it.effect.each(["manual_shutdown", "server_shutdown"] as const)(
2904
+ "ProviderSessionManagerV2 preserves questions only for server shutdown (%s)",
2905
+ (reason) =>
2906
+ Effect.gen(function* () {
2907
+ const state = yield* Ref.make(emptyState);
2908
+ const effect = Effect.gen(function* () {
2909
+ const eventSink = yield* EventSink.EventSinkV2;
2910
+ const idAllocator = yield* IdAllocator.IdAllocatorV2;
2911
+ const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
2912
+ const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
2913
+ const now = yield* DateTime.now;
2914
+ const projectId = yield* idAllocator.allocate.project({
2915
+ fixtureName: "provider-session-manager-request-expire",
2916
+ });
2917
+ const threadId = yield* idAllocator.allocate.thread({
2918
+ fixtureName: "provider-session-manager-request-expire",
2919
+ projectId,
2920
+ });
2921
+ const providerSessionId = yield* idAllocator.allocate.providerSession({
2922
+ providerInstanceId: modelSelection.instanceId,
2923
+ threadId,
2924
+ });
2925
+ const providerThread = makeProviderThread({
2926
+ idAllocator,
2927
+ threadId,
2928
+ providerSessionId,
2929
+ now,
2930
+ });
2931
+
2932
+ yield* eventSink.write({
2933
+ events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })],
2934
+ });
2935
+ const pendingRequest = yield* makePendingRuntimeRequestEvents({
2936
+ idAllocator,
2937
+ threadId,
2938
+ providerSessionId,
2939
+ providerThread,
2940
+ now,
2941
+ });
2942
+ yield* eventSink.write({
2943
+ events: pendingRequest.events.map((event) =>
2944
+ event.type === "turn-item.updated"
2945
+ ? {
2946
+ ...event,
2947
+ payload: {
2948
+ ...event.payload,
2949
+ type: "user_input_request",
2950
+ questions: [
2951
+ { id: "color", header: "Color", question: "Which color?", options: [] },
2952
+ ],
2953
+ },
2954
+ }
2955
+ : event.type === "runtime-request.updated"
2956
+ ? { ...event, payload: { ...event.payload, kind: "user_input" } }
2957
+ : event,
2958
+ ),
2959
+ });
2960
+ yield* manager.open({
2961
+ threadId,
2962
+ providerSessionId,
2963
+ modelSelection,
2964
+ runtimePolicy,
2965
+ });
2966
+ yield* manager.release({
2967
+ providerSessionId,
2968
+ reason,
2969
+ });
2970
+
2971
+ const projection = yield* projectionStore.getThreadProjection(threadId);
2972
+ const request = projection.runtimeRequests.at(-1);
2973
+ const requestNode = projection.nodes.find((node) => node.id === request?.nodeId);
2974
+ const requestTurnItem = projection.turnItems.find(
2975
+ (item) => item.type === "user_input_request" && item.requestId === request?.id,
2976
+ );
2977
+
2978
+ const preserved = reason === "server_shutdown";
2979
+ assert.equal(request?.status, preserved ? "pending" : "cancelled");
2980
+ assert.equal(request?.responseCapability.type, preserved ? "live" : "not_resumable");
2981
+ assert.equal(requestNode?.status, preserved ? "waiting" : "cancelled");
2982
+ assert.equal(requestTurnItem?.status, preserved ? "waiting" : "cancelled");
2983
+ });
2984
+
2985
+ yield* effect.pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 1000 })));
2986
+ }),
2987
+);
2988
+
2989
it.effect("ProviderSessionManagerV2 persists session-scoped runtime requests without a run", () =>
2990
Effect.gen(function* () {
2991
const state = yield* Ref.make(emptyState);
apps/server/src/orchestration-v2/ProviderSessionManager.ts
+3
-1
@@ -633,7 +633,9 @@ export const layerWithOptions = (
633
request.status === "pending" &&
634
request.responseCapability.type === "live" &&
635
request.responseCapability.providerSessionId === providerSessionId &&
636
- DateTime.isLessThanOrEqualTo(request.createdAt, input.releasedAt),
636
+ DateTime.isLessThanOrEqualTo(request.createdAt, input.releasedAt) &&
637
+ // Shutdown reconciliation decides which questions can survive the lost callback.
638
+ !(input.reason === "server_shutdown" && request.kind === "user_input"),
639
);
640
641
for (const request of releasedRequests) {
apps/server/src/orchestration-v2/runtimeLayer.test.ts
+315
-93
@@ -18,6 +18,7 @@ import {
18
TurnItemId,
19
type ModelSelection,
20
type OrchestrationV2Run,
21
+ type OrchestrationV2RuntimeRequest,
22
ProjectId,
23
type PullRequestDetail,
24
type PullRequestComment,
@@ -1077,15 +1078,288 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => {
1078
}),
1079
);
1080
1081
+ const seedPendingQuestion = Effect.fnUntraced(function* (
1082
+ threadId: ThreadId,
1083
+ runId: RunId | null,
1084
+ responseCapability: OrchestrationV2RuntimeRequest["responseCapability"],
1085
+ ) {
1086
+ const eventSink = yield* EventSink.EventSinkV2;
1087
+ const now = yield* DateTime.now;
1088
+ const requestId = RuntimeRequestId.make(`${threadId}:request`);
1089
+ const nodeId = NodeId.make(`${threadId}:node`);
1090
+ const itemId = TurnItemId.make(`${threadId}:item`);
1091
+ yield* eventSink.write({
1092
+ commandId: CommandId.make(`${threadId}:seed`),
1093
+ events: [
1094
+ {
1095
+ id: EventId.make(`${threadId}:node-event`),
1096
+ type: "node.updated",
1097
+ threadId,
1098
+ nodeId,
1099
+ occurredAt: now,
1100
+ payload: {
1101
+ id: nodeId,
1102
+ threadId,
1103
+ runId,
1104
+ parentNodeId: null,
1105
+ rootNodeId: nodeId,
1106
+ kind: "user_input_request",
1107
+ status: "waiting",
1108
+ countsForRun: false,
1109
+ providerThreadId: null,
1110
+ providerTurnId: null,
1111
+ nativeItemRef: null,
1112
+ runtimeRequestId: requestId,
1113
+ checkpointScopeId: null,
1114
+ startedAt: now,
1115
+ completedAt: null,
1116
+ },
1117
+ },
1118
+ {
1119
+ id: EventId.make(`${threadId}:request-event`),
1120
+ type: "runtime-request.updated",
1121
+ threadId,
1122
+ nodeId,
1123
+ occurredAt: now,
1124
+ payload: {
1125
+ id: requestId,
1126
+ nodeId,
1127
+ providerTurnId: null,
1128
+ nativeRequestRef: null,
1129
+ kind: "user_input",
1130
+ status: "pending",
1131
+ responseCapability,
1132
+ createdAt: now,
1133
+ resolvedAt: null,
1134
+ },
1135
+ },
1136
+ {
1137
+ id: EventId.make(`${threadId}:item-event`),
1138
+ type: "turn-item.updated",
1139
+ threadId,
1140
+ nodeId,
1141
+ occurredAt: now,
1142
+ payload: {
1143
+ id: itemId,
1144
+ type: "user_input_request",
1145
+ threadId,
1146
+ runId,
1147
+ nodeId,
1148
+ providerThreadId: null,
1149
+ providerTurnId: null,
1150
+ nativeItemRef: null,
1151
+ parentItemId: null,
1152
+ ordinal: 0,
1153
+ status: "waiting",
1154
+ title: null,
1155
+ startedAt: now,
1156
+ completedAt: null,
1157
+ updatedAt: now,
1158
+ requestId,
1159
+ questions: [
1160
+ { id: "color", header: "Color", question: "Which color?", options: [] },
1161
+ {
1162
+ id: "sizes",
1163
+ header: "Sizes",
1164
+ question: "Which sizes?",
1165
+ options: [],
1166
+ multiSelect: true,
1167
+ },
1168
+ {
1169
+ id: "note",
1170
+ header: "Note",
1171
+ question: "Anything else?",
1172
+ options: [],
1173
+ required: false,
1174
+ },
1175
+ ],
1176
+ },
1177
+ },
1178
+ ],
1179
+ });
1180
+ return { requestId, nodeId, itemId };
1181
+ });
1182
+
1183
+ it.effect.each([
1184
+ { trigger: "startup", scope: "run" },
1185
+ { trigger: "shutdown", scope: "session" },
1186
+ ] as const)(
1187
+ "recovers only reconstructable user questions and keeps their cards pending on $trigger ($scope)",
1188
+ ({ trigger, scope }) =>
1189
+ Effect.gen(function* () {
1190
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
1191
+ const eventSink = yield* EventSink.EventSinkV2;
1192
+ const now = yield* DateTime.now;
1193
+ const threadId = ThreadId.make(`runtime-recovered-question-${trigger}-${scope}`);
1194
+ yield* orchestrator.dispatch({
1195
+ type: "thread.create",
1196
+ commandId: CommandId.make(`runtime-recovered-question-create-${trigger}-${scope}`),
1197
+ createdBy: "user",
1198
+ creationSource: "web",
1199
+ threadId,
1200
+ projectId: ProjectId.make(`runtime-recovered-question-project-${trigger}-${scope}`),
1201
+ title: "Async question",
1202
+ modelSelection,
1203
+ runtimeMode: "full-access",
1204
+ interactionMode: "default",
1205
+ branch: null,
1206
+ worktreePath: process.cwd(),
1207
+ });
1208
+ if (scope === "run") {
1209
+ yield* orchestrator.dispatch({
1210
+ type: "message.dispatch",
1211
+ commandId: CommandId.make(`${threadId}:run`),
1212
+ threadId,
1213
+ messageId: MessageId.make(`${threadId}:message`),
1214
+ text: "Ask me a question",
1215
+ attachments: [],
1216
+ createdBy: "user",
1217
+ creationSource: "web",
1218
+ dispatchMode: { type: "start_immediately" },
1219
+ });
1220
+ }
1221
+ const runId = (yield* orchestrator.getThreadProjection(threadId)).runs[0]?.id ?? null;
1222
+ const { requestId, nodeId, itemId } = yield* seedPendingQuestion(threadId, runId, {
1223
+ type: "live",
1224
+ providerSessionId: ProviderSessionId.make(`lost-session-${trigger}-${scope}`),
1225
+ });
1226
+ const recovery = yield* ProviderRuntimeRecoveryService.ProviderRuntimeRecoveryService;
1227
+ const original = yield* orchestrator.getThreadProjection(threadId);
1228
+ const request = original.runtimeRequests[0]!;
1229
+ const questionItem = original.turnItems.find((item) => item.type === "user_input_request")!;
1230
+ const nativeThreadId = ThreadId.make(`runtime-recovered-native-child-${trigger}-${scope}`);
1231
+ const nativeRequestId = RuntimeRequestId.make(
1232
+ `runtime-recovered-native-request-${trigger}-${scope}`,
1233
+ );
1234
+ const approvalId = RuntimeRequestId.make(`runtime-recovered-approval-${trigger}-${scope}`);
1235
+ const missingTextId = RuntimeRequestId.make(
1236
+ `runtime-recovered-missing-text-${trigger}-${scope}`,
1237
+ );
1238
+ yield* eventSink.write({
1239
+ events: [
1240
+ {
1241
+ id: EventId.make(`recovery-approval-${trigger}-${scope}`),
1242
+ type: "runtime-request.updated",
1243
+ threadId,
1244
+ occurredAt: now,
1245
+ payload: { ...request, id: approvalId, kind: "command" },
1246
+ },
1247
+ {
1248
+ id: EventId.make(`recovery-missing-text-${trigger}-${scope}`),
1249
+ type: "runtime-request.updated",
1250
+ threadId,
1251
+ occurredAt: now,
1252
+ payload: { ...request, id: missingTextId },
1253
+ },
1254
+ {
1255
+ id: EventId.make(`recovery-native-thread-${trigger}-${scope}`),
1256
+ type: "thread.created",
1257
+ threadId: nativeThreadId,
1258
+ occurredAt: now,
1259
+ payload: {
1260
+ ...original.thread,
1261
+ id: nativeThreadId,
1262
+ creationSource: "provider",
1263
+ lineage: {
1264
+ parentThreadId: threadId,
1265
+ rootThreadId: threadId,
1266
+ relationshipToParent: "subagent",
1267
+ },
1268
+ },
1269
+ },
1270
+ {
1271
+ id: EventId.make(`recovery-native-request-${trigger}-${scope}`),
1272
+ type: "runtime-request.updated",
1273
+ threadId: nativeThreadId,
1274
+ occurredAt: now,
1275
+ payload: { ...request, id: nativeRequestId },
1276
+ },
1277
+ {
1278
+ id: EventId.make(`recovery-native-item-${trigger}-${scope}`),
1279
+ type: "turn-item.updated",
1280
+ threadId: nativeThreadId,
1281
+ occurredAt: now,
1282
+ payload: {
1283
+ ...questionItem,
1284
+ id: TurnItemId.make(`recovery-native-item-${trigger}-${scope}`),
1285
+ type: "user_input_request",
1286
+ threadId: nativeThreadId,
1287
+ runId: null,
1288
+ requestId: nativeRequestId,
1289
+ questions: [
1290
+ { id: "color", header: "Color", question: "Which color?", options: [] },
1291
+ ],
1292
+ },
1293
+ },
1294
+ ],
1295
+ });
1296
+ yield* recovery.reconcile(trigger);
1297
+ const recovered = yield* orchestrator.getThreadProjection(threadId);
1298
+ if (scope === "run") {
1299
+ assert.equal(recovered.runs[0]?.status, "cancelled");
1300
+ } else {
1301
+ assert.isEmpty(recovered.runs);
1302
+ }
1303
+ assert.deepEqual(
1304
+ recovered.runtimeRequests.find((row) => row.id === requestId),
1305
+ {
1306
+ ...request,
1307
+ responseCapability: { type: "message" },
1308
+ },
1309
+ );
1310
+ assert.equal(recovered.nodes.find((row) => row.id === nodeId)?.status, "waiting");
1311
+ assert.equal(recovered.turnItems.find((row) => row.id === itemId)?.status, "waiting");
1312
+ for (const id of [approvalId, missingTextId]) {
1313
+ const closed = recovered.runtimeRequests.find((row) => row.id === id);
1314
+ assert.equal(closed?.status, trigger === "startup" ? "expired" : "cancelled");
1315
+ assert.equal(closed?.responseCapability.type, "not_resumable");
1316
+ }
1317
+ const native = yield* orchestrator.getThreadProjection(nativeThreadId);
1318
+ assert.equal(
1319
+ native.runtimeRequests[0]?.status,
1320
+ trigger === "startup" ? "expired" : "cancelled",
1321
+ );
1322
+ assert.equal(native.runtimeRequests[0]?.responseCapability.type, "not_resumable");
1323
+ yield* recovery.reconcile("shutdown");
1324
+ yield* recovery.reconcile("startup");
1325
+ assert.deepEqual(
1326
+ (yield* orchestrator.getThreadProjection(threadId)).runtimeRequests,
1327
+ recovered.runtimeRequests,
1328
+ );
1329
+ yield* orchestrator.dispatch({
1330
+ type: "runtime-request.respond",
1331
+ commandId: CommandId.make(`recovery-answer-${trigger}-${scope}`),
1332
+ threadId,
1333
+ requestId,
1334
+ answers: { color: "Blue", sizes: ["Small", "Large"] },
1335
+ });
1336
+ const answered = yield* orchestrator.getThreadProjection(threadId);
1337
+ assert.equal(
1338
+ answered.runtimeRequests.find((row) => row.id === requestId)?.status,
1339
+ "resolved",
1340
+ );
1341
+ assert.equal(answered.nodes.find((row) => row.id === nodeId)?.status, "completed");
1342
+ assert.equal(answered.turnItems.find((row) => row.id === itemId)?.status, "completed");
1343
+ assert.lengthOf(answered.runs, recovered.runs.length + 1);
1344
+ assert.equal(answered.messages.at(-1)?.runId, answered.runs.at(-1)?.id);
1345
+ assert.equal(
1346
+ answered.messages.at(-1)?.text,
1347
+ "Which color?\nBlue\n\nWhich sizes?\nSmall, Large",
1348
+ );
1349
+ assert.deepEqual(
1350
+ (yield* (yield* EffectOutbox.EffectOutboxV2).listByCommandId(
1351
+ CommandId.make(`recovery-answer-${trigger}-${scope}`),
1352
+ )).map((effect) => effect.request.type),
1353
+ ["provider-turn.start"],
1354
+ );
1355
+ }),
1356
+ );
1357
+
1358
it.effect("answers an async question after its provider exits and commits the answer once", () =>
1359
Effect.gen(function* () {
1360
const orchestrator = yield* Orchestrator.OrchestratorV2;
1361
const eventSink = yield* EventSink.EventSinkV2;
1084
- const now = yield* DateTime.now;
1362
const threadId = ThreadId.make("runtime-async-question");
1086
- const requestId = RuntimeRequestId.make("runtime-async-question-request");
1087
- const nodeId = NodeId.make("runtime-async-question-node");
1088
- const itemId = TurnItemId.make("runtime-async-question-item");
1363
yield* orchestrator.dispatch({
1364
type: "thread.create",
1365
commandId: CommandId.make("runtime-async-question-create"),
@@ -1100,100 +1374,40 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => {
1374
branch: null,
1375
worktreePath: process.cwd(),
1376
});
1103
- yield* eventSink.write({
1104
- commandId: CommandId.make("runtime-async-question-seed"),
1105
- events: [
1106
- {
1107
- id: EventId.make("runtime-async-question-node-event"),
1108
- type: "node.updated",
1109
- threadId,
1110
- nodeId,
1111
- occurredAt: now,
1112
- payload: {
1113
- id: nodeId,
1114
- threadId,
1115
- runId: null,
1116
- parentNodeId: null,
1117
- rootNodeId: nodeId,
1118
- kind: "user_input_request",
1119
- status: "waiting",
1120
- countsForRun: false,
1121
- providerThreadId: null,
1122
- providerTurnId: null,
1123
- nativeItemRef: null,
1124
- runtimeRequestId: requestId,
1125
- checkpointScopeId: null,
1126
- startedAt: now,
1127
- completedAt: null,
1128
- },
1129
- },
1130
- {
1131
- id: EventId.make("runtime-async-question-request-event"),
1132
- type: "runtime-request.updated",
1133
- threadId,
1134
- nodeId,
1135
- occurredAt: now,
1136
- payload: {
1137
- id: requestId,
1138
- nodeId,
1139
- providerTurnId: null,
1140
- nativeRequestRef: null,
1141
- kind: "user_input",
1142
- status: "pending",
1143
- responseCapability: { type: "message" },
1144
- createdAt: now,
1145
- resolvedAt: null,
1146
- },
1147
- },
1148
- {
1149
- id: EventId.make("runtime-async-question-item-event"),
1150
- type: "turn-item.updated",
1151
- threadId,
1152
- nodeId,
1153
- occurredAt: now,
1154
- payload: {
1155
- id: itemId,
1156
- type: "user_input_request",
1157
- threadId,
1158
- runId: null,
1159
- nodeId,
1160
- providerThreadId: null,
1161
- providerTurnId: null,
1162
- nativeItemRef: null,
1163
- parentItemId: null,
1164
- ordinal: 0,
1165
- status: "waiting",
1166
- title: null,
1167
- startedAt: now,
1168
- completedAt: null,
1169
- updatedAt: now,
1170
- requestId,
1171
- responseMode: "message",
1172
- questions: [{ id: "color", header: "Color", question: "Which color?", options: [] }],
1173
- },
1174
- },
1175
- ],
1377
+ const { requestId, nodeId, itemId } = yield* seedPendingQuestion(threadId, null, {
1378
+ type: "message",
1379
});
1177
- const invalid = yield* orchestrator
1178
- .dispatch({
1179
- type: "runtime-request.respond",
1180
- commandId: CommandId.make("runtime-async-question-blank"),
1181
- threadId,
1182
- requestId,
1183
- answers: { color: " " },
1184
- })
1185
- .pipe(Effect.result);
1186
- assert.equal(invalid._tag, "Failure");
1187
- const unanswered = yield* orchestrator.getThreadProjection(threadId);
1188
- assert.equal(unanswered.runtimeRequests[0]?.status, "pending");
1189
- assert.deepEqual(unanswered.messages, []);
1380
+ const outbox = yield* EffectOutbox.EffectOutboxV2;
1381
+ const beforeInvalid = yield* orchestrator.getThreadProjection(threadId);
1382
+ const beforeSequence = yield* eventSink.latestSequence();
1383
+ for (const [index, answers] of [
1384
+ { color: " ", sizes: ["Small"] },
1385
+ { color: "Blue", sizes: [] },
1386
+ { color: "Blue", sizes: ["Small", " "] },
1387
+ { color: "Blue", sizes: ["Small"], extra: " " },
1388
+ ].entries()) {
1389
+ const commandId = CommandId.make(`runtime-async-question-invalid:${index}`);
1390
+ const invalid = yield* orchestrator
1391
+ .dispatch({
1392
+ type: "runtime-request.respond",
1393
+ commandId,
1394
+ threadId,
1395
+ requestId,
1396
+ answers,
1397
+ })
1398
+ .pipe(Effect.result);
1399
+ assert.equal(invalid._tag, "Failure", JSON.stringify(answers));
1400
+ assert.deepEqual(yield* orchestrator.getThreadProjection(threadId), beforeInvalid);
1401
+ assert.equal(yield* eventSink.latestSequence(), beforeSequence);
1402
+ assert.deepEqual(yield* outbox.listByCommandId(commandId), []);
1403
+ }
1404
1405
const command = {
1406
type: "runtime-request.respond" as const,
1407
commandId: CommandId.make("runtime-async-question-answer"),
1408
threadId,
1409
requestId,
1196
- answers: { color: " Blue " },
1410
+ answers: { color: " Blue ", sizes: ["Small", "Large"], note: "" },
1411
};
1412
const accepted = yield* orchestrator.dispatch(command);
1413
const repeated = yield* orchestrator.dispatch(command);
@@ -1210,13 +1424,21 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => {
1424
requestId,
1425
answers: command.answers,
1426
attachmentsByQuestionId: {},
1213
- questionTextById: { color: "Which color?" },
1427
+ questionTextById: {
1428
+ color: "Which color?",
1429
+ sizes: "Which sizes?",
1430
+ note: "Anything else?",
1431
+ },
1432
});
1433
}
1434
assert.equal(answered.messages.length, 1);
1217
- assert.equal(answered.messages[0]?.text, "Which color?\nBlue");
1435
+ assert.equal(answered.messages[0]?.text, "Which color?\nBlue\n\nWhich sizes?\nSmall, Large");
1436
assert.equal(answered.messages[0]?.role, "user");
1437
assert.equal(answered.runs.length, 1);
1438
+ assert.deepEqual(
1439
+ (yield* outbox.listByCommandId(command.commandId)).map((effect) => effect.request.type),
1440
+ ["provider-turn.start"],
1441
+ );
1442
1443
const duplicate = yield* orchestrator
1444
.dispatch({