The best way to control Claude Code, Codex and more from any computer or phone
perf(server): busy threads stop decoding settled subagents and idle pollers skip work (#27711)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Julius Marminge committed
Oct 30, 2026 at 12:22 UTC
9b1d351ba0f63f527f884ffb3f2e394f50388764
19 files changed
+645
-92
apps/server/src/mcp/OrchestratorMcpService.ts
+48
-9
@@ -513,6 +513,21 @@ function pendingTaskApproval(
513
return approval;
514
}
515
516
+/** Every task `directAppOwnedChildTasks` can return, so callers load only those. */
517
+function directChildTaskIds(
518
+ target: Pick<OrchestrationV2ThreadProjection, "thread" | "runs">,
519
+ returnedRunIds: ReadonlySet<RunId | null>,
520
+): ReadonlyArray<NodeId> {
521
+ const ids = new Set<NodeId>();
522
+ if (target.thread.forkedFrom?.type === "node") ids.add(target.thread.forkedFrom.nodeId);
523
+ for (const run of target.runs) {
524
+ if (returnedRunIds.has(run.id) && run.delegatedTaskId !== undefined) {
525
+ ids.add(run.delegatedTaskId);
526
+ }
527
+ }
528
+ return [...ids];
529
+}
530
+
531
function directAppOwnedChildTasks(
532
parent: Pick<OrchestrationV2ThreadProjection, "thread" | "subagents">,
533
target: Pick<OrchestrationV2ThreadProjection, "thread" | "runs">,
@@ -1016,7 +1031,6 @@ const make = Effect.gen(function* () {
1031
[
1032
"runs",
1033
"messages",
1019
- "subagents",
1034
"providerThreads",
1035
"providerTurns",
1036
"runtimeRequests",
@@ -1034,6 +1048,15 @@ const make = Effect.gen(function* () {
1048
),
1049
);
1050
1051
+ // A dispatcher thread holds thousands of tasks, each with its full prompt; read only the ones named.
1052
+ const loadTasks = (threadId: ThreadId, taskIds: ReadonlyArray<NodeId>) =>
1053
+ threadManagement.getThreadRecords(threadId, ["subagents"], { subagentIds: taskIds }).pipe(
1054
+ Effect.map(({ subagents }) => subagents),
1055
+ Effect.mapError((error) =>
1056
+ failure("orchestration_error", `Unable to read thread ${threadId}: ${errorMessage(error)}`),
1057
+ ),
1058
+ );
1059
+
1060
// Task waits read on every wake of a parent that may hold thousands of
1061
// tasks, so they read only the task they watch and what its status needs.
1062
const loadTaskParent = (threadId: ThreadId, taskId: NodeId) =>
@@ -2385,7 +2408,7 @@ const make = Effect.gen(function* () {
2408
const childThreadId = current.childThreadId;
2409
if (childThreadId === null) {
2410
// The task runs in a linked environment: its thread there is stopped through the link.
2388
- const { subagents } = yield* loadProjection(scope.thread.threadId);
2411
+ const subagents = yield* loadTasks(scope.thread.threadId, [input.taskId]);
2412
const remoteTask = subagents.find(
2413
(task) =>
2414
task.id === input.taskId &&
@@ -2428,7 +2451,7 @@ const make = Effect.gen(function* () {
2451
}
2452
});
2453
yield* assertStoppable(childThreadId);
2431
- const parentTask = parentProjection.subagents.find(
2454
+ const parentTask = (yield* loadTasks(scope.thread.threadId, [input.taskId])).find(
2455
(task) => task.id === input.taskId && task.origin === "app_owned",
2456
);
2457
const disposeCompletionDelivery =
@@ -2548,9 +2571,11 @@ const make = Effect.gen(function* () {
2571
runId: parentTask.runId,
2572
subagentId: parentTask.id,
2573
});
2551
- const { subagents } = yield* threadManagement.getThreadRecords(scope.thread.threadId, [
2552
- "subagents",
2553
- ]);
2574
+ const { subagents } = yield* threadManagement.getThreadRecords(
2575
+ scope.thread.threadId,
2576
+ ["subagents"],
2577
+ { subagentIds: [input.taskId] },
2578
+ );
2579
return subagents.some((task) => task.id === input.taskId && task.status === "cancelled");
2580
}).pipe(Effect.mapError(threadManagementFailure));
2581
if (isTerminalTaskStatus(current.status)) {
@@ -2847,7 +2872,19 @@ const make = Effect.gen(function* () {
2872
page.filter((row) => row.sourceThreadId === target.thread.id).map((row) => row.item.runId),
2873
);
2874
const tasks =
2850
- parent === undefined ? [] : directAppOwnedChildTasks(parent, target, returnedRunIds);
2875
+ parent === undefined
2876
+ ? []
2877
+ : directAppOwnedChildTasks(
2878
+ {
2879
+ thread: parent.thread,
2880
+ subagents: yield* loadTasks(
2881
+ parent.thread.id,
2882
+ directChildTaskIds(target, returnedRunIds),
2883
+ ),
2884
+ },
2885
+ target,
2886
+ returnedRunIds,
2887
+ );
2888
if (
2889
parent !== undefined &&
2890
scope.thread !== undefined &&
@@ -2978,14 +3015,16 @@ const make = Effect.gen(function* () {
3015
const ownedTask =
3016
result.timedOut &&
3017
status !== "waiting" &&
2981
- parent?.subagents.some(
3018
+ parent !== undefined &&
3019
+ taskId !== undefined &&
3020
+ (yield* loadTasks(parent.thread.id, [taskId])).some(
3021
(task) =>
3022
task.id === taskId &&
3023
task.origin === "app_owned" &&
3024
task.threadId === parent.thread.id &&
3025
task.childThreadId === input.threadId &&
3026
task.completionDelivery?.state !== "disposed",
2988
- ) === true;
3027
+ );
3028
// A child waiting on a question or approval needs an answer, not a wake.
3029
const childRequests = ownedTask
3030
? (yield* threadManagement
apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts
+3
-2
@@ -4944,7 +4944,8 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
4944
});
4945
4946
// Claude has no recovered frame; the next assistant message proves recovery.
4947
- const completeProviderRetry = Effect.fn("ClaudeAdapterV2.completeProviderRetry")(function* (
4947
+ // Runs on every model output frame and is almost always a no-op; only a pending retry gets a span.
4948
+ const completeProviderRetry = Effect.fnUntraced(function* (
4949
context: ActiveClaudeTurnContext,
4950
updatedAt: DateTime.Utc,
4951
) {
@@ -4977,7 +4978,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
4978
startedAt: providerRetry.startedAt,
4979
updatedAt,
4980
}),
4980
- });
4981
+ }).pipe(Effect.withSpan("ClaudeAdapterV2.completeProviderRetry"));
4982
});
4983
4984
const rememberProviderThread =
apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts
+3
-2
@@ -1945,7 +1945,8 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
1945
});
1946
1947
// Only new model output proves recovery; local item/completed can arrive during upstream retries.
1948
- const completeProviderRetry = Effect.fn("CodexAdapterV2.completeProviderRetry")(function* (
1948
+ // Runs on every model output frame and is almost always a no-op; only a pending retry gets a span.
1949
+ const completeProviderRetry = Effect.fnUntraced(function* (
1950
context: ActiveCodexTurnContext,
1951
updatedAt: DateTime.Utc,
1952
) {
@@ -1965,7 +1966,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
1966
type: "turn_item.updated",
1967
driver: CODEX_PROVIDER,
1968
turnItem: codexRetryTurnItem(context, providerRetry, "completed", updatedAt),
1968
- });
1969
+ }).pipe(Effect.withSpan("CodexAdapterV2.completeProviderRetry"));
1970
});
1971
1972
const registerRootTurn = Effect.fn("CodexAdapterV2.registerRootTurn")(function* (input: {
apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts
+72
-13
@@ -159,6 +159,11 @@ const seedParentWithTerminalTask = (input: {
159
readonly deliveryState: "delivered" | "claimed" | "acknowledged" | "disposed";
160
readonly completionWake?: "always" | "settled_only";
161
readonly deliveryTaskIds?: ReadonlyArray<NodeId>;
162
+ readonly parentRunStatus?: "running" | "completed";
163
+ readonly siblingTasks?: ReadonlyArray<{
164
+ readonly id: NodeId;
165
+ readonly deliveryState: "delivered" | "claimed";
166
+ }>;
167
readonly now: DateTime.Utc;
168
}) =>
169
Effect.gen(function* () {
@@ -241,10 +246,10 @@ const seedParentWithTerminalTask = (input: {
246
userMessageId: MessageId.make(`message:seed-user:${input.threadId}`),
247
rootNodeId: input.rootNodeId,
248
activeAttemptId: null,
244
- status: "running",
249
+ status: input.parentRunStatus ?? "running",
250
requestedAt: input.now,
251
startedAt: input.now,
247
- completedAt: null,
252
+ completedAt: input.parentRunStatus === "completed" ? input.now : null,
253
checkpointId: null,
254
contextHandoffId: null,
255
delegatedCompletion: {
@@ -261,22 +266,25 @@ const seedParentWithTerminalTask = (input: {
266
},
267
},
268
},
264
- {
265
- id: EventId.make(`event:seed-task:${input.threadId}`),
266
- type: "subagent.updated",
269
+ ...[
270
+ { id: input.taskId, deliveryState: input.deliveryState },
271
+ ...(input.siblingTasks ?? []),
272
+ ].map((task) => ({
273
+ id: EventId.make(`event:seed-task:${input.threadId}:${task.id}`),
274
+ type: "subagent.updated" as const,
275
threadId: input.threadId,
276
runId: input.runId,
269
- nodeId: input.taskId,
277
+ nodeId: task.id,
278
driver,
279
providerInstanceId: modelSelection.instanceId,
280
occurredAt: input.now,
281
payload: {
274
- id: input.taskId,
282
+ id: task.id,
283
threadId: input.threadId,
284
runId: input.runId,
285
parentNodeId: input.rootNodeId,
278
- origin: "app_owned",
279
- createdBy: "agent",
286
+ origin: "app_owned" as const,
287
+ createdBy: "agent" as const,
288
driver,
289
providerInstanceId: modelSelection.instanceId,
290
providerThreadId: null,
@@ -287,16 +295,16 @@ const seedParentWithTerminalTask = (input: {
295
model: null,
296
completionWake: input.completionWake ?? "settled_only",
297
completionDelivery: {
290
- state: input.deliveryState,
291
- observedByRunId: input.deliveryState === "acknowledged" ? input.runId : null,
298
+ state: task.deliveryState,
299
+ observedByRunId: task.deliveryState === "acknowledged" ? input.runId : null,
300
},
293
- status: "completed",
301
+ status: "completed" as const,
302
result: "child finished",
303
startedAt: input.now,
304
completedAt: input.now,
305
updatedAt: input.now,
306
},
299
- },
307
+ })),
308
],
309
});
310
});
@@ -683,6 +691,57 @@ it.layer(layerTest)("delegated completion delivery repairs", (it) => {
691
}),
692
);
693
694
+ it.effect("counts already delivered siblings in the completion notice", () =>
695
+ Effect.gen(function* () {
696
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
697
+ const now = yield* DateTime.now;
698
+ const threadId = ThreadId.make("thread:delegated-delivery-notice-count");
699
+ const runId = RunId.make("run:delegated-delivery-notice-count");
700
+ const firstTaskId = NodeId.make("node:delegated-delivery-notice-count-first");
701
+ const secondTaskId = NodeId.make("node:delegated-delivery-notice-count-second");
702
+
703
+ yield* seedParentWithTerminalTask({
704
+ threadId,
705
+ projectId: ProjectId.make("project:delegated-delivery-notice-count"),
706
+ runId,
707
+ rootNodeId: NodeId.make("node:delegated-delivery-notice-count-root"),
708
+ taskId: firstTaskId,
709
+ deliveryState: "claimed",
710
+ completionWake: "always",
711
+ deliveryTaskIds: [firstTaskId, secondTaskId],
712
+ parentRunStatus: "completed",
713
+ siblingTasks: [
714
+ { id: secondTaskId, deliveryState: "claimed" },
715
+ {
716
+ id: NodeId.make("node:delegated-delivery-notice-count-earlier"),
717
+ deliveryState: "delivered",
718
+ },
719
+ ],
720
+ now,
721
+ });
722
+
723
+ yield* orchestrator.dispatch({
724
+ type: "message.dispatch",
725
+ commandId: CommandId.make("command:delegated-delivery-notice-count"),
726
+ threadId,
727
+ messageId: MessageId.make(`message:delegated-delivery:${threadId}`),
728
+ text: "Delegated tasks reached a terminal state.",
729
+ attachments: [],
730
+ dispatchMode: { type: "queue_after_active" },
731
+ createdBy: "agent",
732
+ creationSource: "server",
733
+ delegatedCompletion: { parentRunId: runId, generation: 1, taskIds: [firstTaskId] },
734
+ });
735
+
736
+ const projection = yield* orchestrator.getThreadProjection(threadId);
737
+ const notice = projection.turnItems.find((item) => item.type === "notification");
738
+ assert.match(
739
+ notice?.type === "notification" ? notice.summary : "",
740
+ /^2 of 3 delegated tasks finished:/,
741
+ );
742
+ }),
743
+ );
744
+
745
it.effect.each([
746
{
747
name: "oversized UTF-8 result",
apps/server/src/orchestration-v2/Orchestrator.ts
+74
-24
@@ -1153,6 +1153,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1153
turnItemTypes: ["user_message", "error"],
1154
messageRoles: ["user"],
1155
omitSettledLeafNodes: true,
1156
+ // Commands here read only undelivered or active tasks; a dispatcher thread keeps
1157
+ // thousands of delivered ones, each carrying its full prompt.
1158
+ omitSettledSubagents: true,
1159
},
1160
)
1161
.pipe(
@@ -1167,6 +1170,23 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1170
Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })),
1171
);
1172
1173
+ // Command reads omit settled tasks, but a completion notice counts its whole cohort.
1174
+ const notificationTasks = (
1175
+ threadId: ThreadId,
1176
+ message: Pick<OrchestrationV2ConversationMessage, "delegatedCompletion">,
1177
+ loaded: ReadonlyArray<OrchestrationV2Subagent>,
1178
+ ) =>
1179
+ message.delegatedCompletion === undefined
1180
+ ? Effect.succeed(loaded)
1181
+ : projectionStore
1182
+ .getThreadRecords(threadId, ["subagents"], {
1183
+ subagentRunIds: [message.delegatedCompletion.parentRunId],
1184
+ })
1185
+ .pipe(
1186
+ Effect.map((records) => records.subagents),
1187
+ Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })),
1188
+ );
1189
+
1190
const readHandoffItems = (threadId: ThreadId, runIds?: ReadonlyArray<RunId | null>) =>
1191
projectionStore
1192
.getThreadRecords(
@@ -2174,7 +2194,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
2194
nodeId: rootNodeId,
2195
providerInstanceId: queuedRun.providerInstanceId,
2196
occurredAt: now,
2177
- payload: notificationTurnItem(userTurnItem, queuedMessage, projection.subagents),
2197
+ payload: notificationTurnItem(
2198
+ userTurnItem,
2199
+ queuedMessage,
2200
+ yield* notificationTasks(threadId, queuedMessage, projection.subagents),
2201
+ ),
2202
},
2203
{
2204
type: "run.updated",
@@ -4677,7 +4701,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
4701
nodeId: messageInput.nodeId,
4702
providerInstanceId: messageInput.providerInstanceId,
4703
occurredAt: now,
4680
- payload: notificationTurnItem(turnItem, message, input.projection.subagents),
4704
+ payload: notificationTurnItem(
4705
+ turnItem,
4706
+ message,
4707
+ yield* notificationTasks(input.command.threadId, message, input.projection.subagents),
4708
+ ),
4709
});
4710
},
4711
);
@@ -6328,7 +6356,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
6356
nodeId: rootNodeId,
6357
providerInstanceId: modelSelection.instanceId,
6358
occurredAt: now,
6331
- payload: notificationTurnItem(turnItem, message, projection.subagents),
6359
+ payload: notificationTurnItem(
6360
+ turnItem,
6361
+ message,
6362
+ yield* notificationTasks(command.threadId, message, projection.subagents),
6363
+ ),
6364
});
6365
if (preparationTurnItem !== null) {
6366
yield* emitEvent({
@@ -7222,7 +7254,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
7254
nodeId: rootNodeId,
7255
providerInstanceId: modelSelection.instanceId,
7256
occurredAt: now,
7225
- payload: notificationTurnItem(turnItem, message, projection.subagents),
7257
+ payload: notificationTurnItem(
7258
+ turnItem,
7259
+ message,
7260
+ yield* notificationTasks(command.threadId, message, projection.subagents),
7261
+ ),
7262
});
7263
const forkResolution = nativeForkResolution ?? portableForkResolution;
7264
if (pendingForkTransfer !== undefined && forkResolution !== null) {
@@ -7289,14 +7325,12 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
7325
effects: Ref.Ref<Array<PendingOrchestrationEffectV2>>,
7326
) {
7327
const parentProjection = yield* projectionStore
7292
- .getThreadRecords(command.parentThreadId, [
7293
- "runs",
7294
- "nodes",
7295
- "subagents",
7296
- "providerThreads",
7297
- "providerTurns",
7298
- "attempts",
7299
- ])
7328
+ .getThreadRecords(
7329
+ command.parentThreadId,
7330
+ ["runs", "nodes", "providerThreads", "providerTurns", "attempts"],
7331
+ // The parent node must belong to the active run, whose nodes this keeps.
7332
+ { omitSettledLeafNodes: true },
7333
+ )
7334
.pipe(
7335
Effect.mapError(
7336
(cause) =>
@@ -7366,7 +7400,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
7400
parentTitle: parentProjection.thread.title,
7401
prompt: command.task,
7402
...(command.title === undefined ? {} : { title: command.title }),
7369
- ordinal: parentProjection.subagents.length + 1,
7403
+ ordinal: (yield* subagentCount(command.parentThreadId)) + 1,
7404
});
7405
const childThread: OrchestrationV2AppThread = {
7406
...makeSubagentChildThread({
@@ -7601,7 +7635,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
7635
);
7636
if (run === undefined || run.delegatedTaskId !== undefined) return;
7637
const parent = yield* projectionStore
7604
- .getThreadRecords(parentThreadId, ["runs", "subagents", "providerTurns", "attempts"])
7638
+ .getThreadRecords(parentThreadId, ["runs", "subagents", "providerTurns", "attempts"], {
7639
+ subagentIds: child.forkedFrom?.type === "node" ? [child.forkedFrom.nodeId] : [],
7640
+ })
7641
.pipe(mapDispatchError(command));
7642
const original = parent.subagents.find(
7643
(task) =>
@@ -7729,13 +7765,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
7765
events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
7766
) {
7767
const parentProjection = yield* projectionStore
7732
- .getThreadRecords(command.parentThreadId, [
7733
- "runs",
7734
- "nodes",
7735
- "subagents",
7736
- "providerTurns",
7737
- "attempts",
7738
- ])
7768
+ .getThreadRecords(command.parentThreadId, ["runs", "providerTurns", "attempts"])
7769
.pipe(
7770
Effect.mapError(
7771
(cause) => new OrchestratorProjectionError({ threadId: command.parentThreadId, cause }),
@@ -7767,7 +7797,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
7797
parentTitle: parentProjection.thread.title,
7798
prompt: command.task,
7799
...(command.title === undefined ? {} : { title: command.title }),
7770
- ordinal: parentProjection.subagents.length + 1,
7800
+ ordinal: (yield* subagentCount(command.parentThreadId)) + 1,
7801
});
7802
const common = {
7803
threadId: command.parentThreadId,
@@ -7872,11 +7902,19 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
7902
>,
7903
events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
7904
) {
7905
+ // Delivery planning reads only this task and the undelivered tasks of the queued delivery;
7906
+ // the task's own node and turn item are all it updates besides.
7907
const parentProjection = yield* projectionStore
7908
.getThreadRecords(
7909
command.parentThreadId,
7910
["runs", "messages", "subagents", "providerTurns", "nodes", "turnItems"],
7879
- { turnItemTypes: ["subagent"], messageRoles: ["user"] },
7911
+ {
7912
+ turnItemTypes: ["subagent"],
7913
+ messageRoles: ["user"],
7914
+ omitSettledSubagents: { except: [command.taskId] },
7915
+ nodeIds: [command.taskId],
7916
+ turnItemNodeIds: [command.taskId],
7917
+ },
7918
)
7919
.pipe(
7920
Effect.mapError(
@@ -8102,8 +8140,15 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
8140
command: Extract<OrchestrationV2Command, { readonly type: "delegated_task.wake-policy" }>,
8141
events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
8142
) {
8143
+ // Delivery planning reads only this task and the undelivered tasks of the queued delivery.
8144
const parentProjection = yield* projectionStore
8106
- .getThreadRecords(command.parentThreadId, ["subagents", "runs", "messages", "providerTurns"])
8145
+ .getThreadRecords(
8146
+ command.parentThreadId,
8147
+ ["subagents", "runs", "messages", "providerTurns"],
8148
+ {
8149
+ omitSettledSubagents: { except: [command.taskId] },
8150
+ },
8151
+ )
8152
.pipe(
8153
Effect.mapError(
8154
(cause) =>
@@ -9546,6 +9591,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
9591
}
9592
});
9593
9594
+ const subagentCount = (threadId: ThreadId) =>
9595
+ projectionStore
9596
+ .getSubagentCount(threadId)
9597
+ .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })));
9598
+
9599
const loadProjectionForCommand = <K extends ProjectionRecordField>(
9600
command: OrchestrationV2Command,
9601
fields: ReadonlyArray<K>,
apps/server/src/orchestration-v2/ProjectionStore.test.ts
+128
@@ -850,6 +850,121 @@ const commandReadsSkipSettledLeafNodes = Effect.gen(function* () {
850
nodeNames(yield* store.getThreadRecords(threadId, ["nodes"])),
851
nodes.map(([name]) => name).toSorted(),
852
);
853
+
854
+ const subagents = [
855
+ ["delivered", settledRun, "completed", "delivered"],
856
+ ["acknowledged", settledRun, "failed", "acknowledged"],
857
+ ["disposed", settledRun, "cancelled", "disposed"],
858
+ ["pending-delivery", settledRun, "completed", "pending"],
859
+ ["claimed", settledRun, "completed", "claimed"],
860
+ ["no-delivery", settledRun, "interrupted", null],
861
+ ["running", settledRun, "running", null],
862
+ ["active-run-delivered", activeRun, "completed", "delivered"],
863
+ ] as const;
864
+ for (const [index, [name, run, status, delivery]] of subagents.entries()) {
865
+ const id = NodeId.make(`node:settled-subagents:${name}`);
866
+ yield* store.apply({
867
+ id: EventId.make(`event:settled-subagents:item:${name}`),
868
+ type: "turn-item.updated",
869
+ threadId,
870
+ runId: run.id,
871
+ nodeId: id,
872
+ driver,
873
+ occurredAt: now,
874
+ payload: {
875
+ id: TurnItemId.make(`item:settled-subagents:${name}`),
876
+ type: "subagent",
877
+ threadId,
878
+ runId: run.id,
879
+ nodeId: id,
880
+ providerThreadId: null,
881
+ providerTurnId: null,
882
+ nativeItemRef: null,
883
+ parentItemId: null,
884
+ ordinal: 100 + index,
885
+ status: status === "running" ? "running" : "completed",
886
+ title: name,
887
+ startedAt: now,
888
+ completedAt: status === "running" ? null : now,
889
+ updatedAt: now,
890
+ subagentId: id,
891
+ origin: "app_owned",
892
+ driver,
893
+ providerInstanceId,
894
+ childThreadId: null,
895
+ prompt: "Do the work",
896
+ result: null,
897
+ },
898
+ });
899
+ yield* store.apply({
900
+ id: EventId.make(`event:settled-subagents:${name}`),
901
+ type: "subagent.updated",
902
+ threadId,
903
+ runId: run.id,
904
+ nodeId: id,
905
+ driver,
906
+ providerInstanceId,
907
+ occurredAt: now,
908
+ payload: {
909
+ id,
910
+ threadId,
911
+ runId: run.id,
912
+ parentNodeId: run.rootNodeId!,
913
+ origin: "app_owned",
914
+ createdBy: "agent",
915
+ driver,
916
+ providerInstanceId,
917
+ providerThreadId: null,
918
+ childThreadId: null,
919
+ nativeTaskRef: null,
920
+ prompt: "Do the work",
921
+ title: name,
922
+ model: null,
923
+ ...(delivery === null
924
+ ? {}
925
+ : { completionDelivery: { state: delivery, observedByRunId: null } }),
926
+ status,
927
+ result: null,
928
+ startedAt: now,
929
+ completedAt: status === "running" ? null : now,
930
+ updatedAt: now,
931
+ },
932
+ });
933
+ }
934
+ const subagentNames = (records: {
935
+ readonly subagents: ReadonlyArray<{ readonly title: string | null }>;
936
+ }) => records.subagents.map((subagent) => subagent.title ?? "").toSorted();
937
+ assert.deepEqual(
938
+ subagentNames(
939
+ yield* store.getThreadRecords(threadId, ["subagents"], { omitSettledSubagents: true }),
940
+ ),
941
+ ["active-run-delivered", "claimed", "no-delivery", "pending-delivery", "running"],
942
+ );
943
+ assert.deepEqual(
944
+ subagentNames(yield* store.getThreadRecords(threadId, ["subagents"])),
945
+ subagents.map(([name]) => name).toSorted(),
946
+ );
947
+ // A command about one settled task keeps it, and reads only its node and turn item.
948
+ const deliveredId = NodeId.make("node:settled-subagents:delivered");
949
+ const forTask = yield* store.getThreadRecords(threadId, ["subagents", "nodes", "turnItems"], {
950
+ omitSettledSubagents: { except: [deliveredId] },
951
+ nodeIds: [NodeId.make("node:settled-leaf-nodes:settled-tool")],
952
+ turnItemNodeIds: [deliveredId],
953
+ });
954
+ assert.deepEqual(subagentNames(forTask), [
955
+ "active-run-delivered",
956
+ "claimed",
957
+ "delivered",
958
+ "no-delivery",
959
+ "pending-delivery",
960
+ "running",
961
+ ]);
962
+ assert.deepEqual(nodeNames(forTask), ["settled-tool"]);
963
+ assert.deepEqual(
964
+ forTask.turnItems.map((item) => item.title),
965
+ ["delivered"],
966
+ );
967
+ assert.strictEqual(yield* store.getSubagentCount(threadId), subagents.length);
968
});
969
970
const backgroundThreadUpdatesKeepTimestamp = Effect.gen(function* () {
@@ -3535,6 +3650,12 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
3650
const projection = yield* projectionStore.getThreadProjection(threadId);
3651
assert.deepEqual(projection.runs[0]?.delegatedCompletion, run.delegatedCompletion);
3652
assert.deepEqual(projection.subagents[0]?.completionDelivery, task.completionDelivery);
3653
+ assert.deepEqual(
3654
+ (yield* projectionStore.getThreadRecords(threadId, ["subagents"], {
3655
+ omitSettledSubagents: true,
3656
+ })).subagents,
3657
+ [],
3658
+ );
3659
}),
3660
);
3661
@@ -4428,6 +4549,8 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
4549
(row) => row.id === threadId,
4550
),
4551
);
4552
+ assert.strictEqual(yield* store.getLimitRecoveryWakeAt(now), DateTime.toEpochMillis(reset));
4553
+ assert.isNull(yield* store.getLimitRecoveryWakeAt(reset));
4554
const due = (yield* store.getLimitRecoveryCandidates({
4555
...recoveryOptions,
4556
now: reset,
@@ -4450,9 +4573,14 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
4573
(row) => row.id === threadId,
4574
),
4575
);
4576
+ assert.strictEqual(
4577
+ yield* store.getLimitRecoveryWakeAt(reset),
4578
+ DateTime.toEpochMillis(DateTime.add(reset, { minutes: 1 })),
4579
+ );
4580
yield* sql`UPDATE orchestration_v2_projection_threads
4581
SET payload_json = json_set(payload_json, '$.snoozedUntil', NULL, '$.limitRecovery.autoResume', json('false'))
4582
WHERE thread_id = ${threadId}`;
4583
+ assert.isNull(yield* store.getLimitRecoveryWakeAt(now));
4584
assert.isUndefined(
4585
(yield* store.getLimitRecoveryCandidates({
4586
...recoveryOptions,
apps/server/src/orchestration-v2/ProjectionStore.ts
+88
-4
@@ -332,11 +332,19 @@ export interface ProjectionRecordFilter {
332
readonly messageIds?: ReadonlyArray<MessageId>;
333
readonly messageRunIds?: ReadonlyArray<RunId>;
334
readonly turnItemRunIds?: ReadonlyArray<RunId | null>;
335
+ readonly turnItemNodeIds?: ReadonlyArray<NodeId>;
336
readonly runIds?: ReadonlyArray<RunId>;
337
readonly subagentIds?: ReadonlyArray<NodeId>;
338
+ readonly subagentRunIds?: ReadonlyArray<RunId>;
339
+ readonly nodeIds?: ReadonlyArray<NodeId>;
340
readonly turnItemTypes?: ReadonlyArray<OrchestrationV2TurnItem["type"]>;
341
readonly turnItemStatuses?: ReadonlyArray<OrchestrationV2TurnItem["status"]>;
342
readonly omitSettledLeafNodes?: boolean;
343
+ /**
344
+ * Skips ended subagents whose completion was acknowledged, delivered or disposed, unless their
345
+ * run is active or `except` names them.
346
+ */
347
+ readonly omitSettledSubagents?: true | { readonly except: ReadonlyArray<NodeId> };
348
}
349
350
// A fork inherits its source's history before it has messages of its own,
@@ -436,6 +444,7 @@ export interface ProjectionStoreV2Shape {
444
options: ProjectionTimelinePageOptions,
445
) => Effect.Effect<ProjectionTimelinePage, ProjectionStoreV2Error>;
446
readonly getMessageCount: (threadId: ThreadId) => Effect.Effect<number, ProjectionStoreV2Error>;
447
+ readonly getSubagentCount: (threadId: ThreadId) => Effect.Effect<number, ProjectionStoreV2Error>;
448
readonly getNextTurnItemOrdinal: (
449
threadId: ThreadId,
450
) => Effect.Effect<number, ProjectionStoreV2Error>;
@@ -496,6 +505,10 @@ export interface ProjectionStoreV2Shape {
505
readonly autoResume: boolean;
506
readonly snooze: boolean;
507
}) => Effect.Effect<ReadonlyArray<ProjectionLimitRecoveryCandidate>, ProjectionStoreV2Error>;
508
+ /** Epoch ms of the next armed reset or snooze end after `now`, when a waiting thread becomes a candidate. */
509
+ readonly getLimitRecoveryWakeAt: (
510
+ now: DateTime.Utc,
511
+ ) => Effect.Effect<number | null, ProjectionStoreV2Error>;
512
readonly getSettlementCandidates: (
513
threadId?: ThreadId,
514
) => Effect.Effect<ReadonlyArray<ProjectionSettlementCandidate>, ProjectionStoreV2Error>;
@@ -2107,6 +2120,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
2120
started_at,
2121
completed_at,
2122
updated_at,
2123
+ completion_delivery_state,
2124
payload_json
2125
)
2126
VALUES (
@@ -2124,6 +2138,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
2138
${nullableStringField(payload, "startedAt")},
2139
${nullableStringField(payload, "completedAt")},
2140
${stringField(payload, "updatedAt")},
2141
+ ${event.payload.completionDelivery?.state ?? null},
2142
${payloadJson}
2143
)
2144
ON CONFLICT(subagent_id)
@@ -2141,6 +2156,10 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
2156
started_at = excluded.started_at,
2157
completed_at = excluded.completed_at,
2158
updated_at = excluded.updated_at,
2159
+ completion_delivery_state = COALESCE(
2160
+ excluded.completion_delivery_state,
2161
+ orchestration_v2_projection_subagents.completion_delivery_state
2162
+ ),
2163
payload_json = CASE
2164
WHEN json_type(excluded.payload_json, '$.completionDelivery') IS NULL
2165
AND json_type(orchestration_v2_projection_subagents.payload_json, '$.completionDelivery') IS NOT NULL
@@ -2980,7 +2999,12 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
2999
const boundedTurnItemRows: ReadonlyArray<WindowPayloadRow> =
3000
(fields !== undefined && !fields.includes("turnItems")) ||
3001
(window === undefined &&
2983
- matchesNothing(filter?.turnItemTypes, filter?.turnItemStatuses, filter?.turnItemRunIds))
3002
+ matchesNothing(
3003
+ filter?.turnItemTypes,
3004
+ filter?.turnItemStatuses,
3005
+ filter?.turnItemRunIds,
3006
+ filter?.turnItemNodeIds,
3007
+ ))
3008
? []
3009
: window === undefined
3010
? yield* sql<PayloadRow>`
@@ -2990,6 +3014,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
3014
${filter?.turnItemTypes === undefined ? sql`` : sql`AND type IN (SELECT value FROM json_each(${encodeIdList(filter.turnItemTypes)}))`}
3015
${filter?.turnItemStatuses === undefined ? sql`` : sql`AND status IN (SELECT value FROM json_each(${encodeIdList(filter.turnItemStatuses)}))`}
3016
${filter?.turnItemRunId === undefined ? sql`` : sql`AND run_id = ${filter.turnItemRunId}`}
3017
+ ${filter?.turnItemNodeIds === undefined ? sql`` : sql`AND node_id IN (SELECT value FROM json_each(${encodeIdList(filter.turnItemNodeIds)}))`}
3018
${filter?.turnItemRunIds === undefined ? sql`` : sql`AND (run_id IN (SELECT value FROM json_each(${encodeIdList(filter.turnItemRunIds.filter((id): id is RunId => id !== null))})) OR (${filter.turnItemRunIds.includes(null) ? 1 : 0} = 1 AND run_id IS NULL))`}
3019
ORDER BY ordinal ASC, turn_item_id ASC
3020
`
@@ -3216,13 +3241,15 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
3241
OR run_id IN (SELECT value FROM json_each(${cohortRunIds})))
3242
ORDER BY run_id ASC, attempt_ordinal ASC
3243
`,
3219
- fields !== undefined && !fields.includes("nodes")
3244
+ (fields !== undefined && !fields.includes("nodes")) ||
3245
+ (window === undefined && matchesNothing(filter?.nodeIds))
3246
? Effect.succeed([])
3247
: window === undefined
3248
? sql<PayloadRow>`
3249
SELECT payload_json
3250
FROM orchestration_v2_projection_nodes AS node
3251
WHERE node.thread_id = ${threadId}
3252
+ ${filter?.nodeIds === undefined ? sql`` : sql`AND node.node_id IN (SELECT value FROM json_each(${encodeIdList(filter.nodeIds)}))`}
3253
${
3254
filter?.omitSettledLeafNodes === true
3255
? sql`AND NOT (
@@ -3253,14 +3280,34 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
3280
),
3281
),
3282
(fields !== undefined && !fields.includes("subagents")) ||
3256
- (window === undefined && matchesNothing(filter?.subagentIds))
3283
+ (window === undefined &&
3284
+ (matchesNothing(filter?.subagentIds) || matchesNothing(filter?.subagentRunIds)))
3285
? Effect.succeed([])
3286
: window === undefined
3287
? sql<PayloadRow>`
3288
SELECT payload_json
3261
- FROM orchestration_v2_projection_subagents
3289
+ FROM orchestration_v2_projection_subagents AS subagent
3290
WHERE thread_id = ${threadId}
3291
${filter?.subagentIds === undefined ? sql`` : sql`AND subagent_id IN (SELECT value FROM json_each(${encodeIdList(filter.subagentIds)}))`}
3292
+ ${filter?.subagentRunIds === undefined ? sql`` : sql`AND run_id IN (SELECT value FROM json_each(${encodeIdList(filter.subagentRunIds)}))`}
3293
+ ${
3294
+ filter?.omitSettledSubagents !== undefined
3295
+ ? sql`AND (${
3296
+ filter.omitSettledSubagents === true
3297
+ ? sql`0`
3298
+ : sql`subagent_id IN (SELECT value FROM json_each(${encodeIdList(filter.omitSettledSubagents.except)}))`
3299
+ } OR NOT (
3300
+ subagent.status IN ('completed', 'failed', 'cancelled', 'interrupted')
3301
+ AND COALESCE(subagent.completion_delivery_state, '')
3302
+ IN ('acknowledged', 'delivered', 'disposed')
3303
+ AND NOT EXISTS (
3304
+ SELECT 1 FROM orchestration_v2_projection_runs AS run
3305
+ WHERE run.run_id = subagent.run_id
3306
+ AND run.status IN ('preparing', 'queued', 'starting', 'running', 'waiting')
3307
+ )
3308
+ ))`
3309
+ : sql``
3310
+ }
3311
ORDER BY COALESCE(started_at, ''), subagent_id ASC
3312
`
3313
: sql<PayloadRow>`
@@ -3863,6 +3910,34 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
3910
Effect.mapError((cause) => new ProjectionStoreSetupError({ cause })),
3911
);
3912
3913
+ const getLimitRecoveryWakeAt = Effect.fn("ProjectionStore.getLimitRecoveryWakeAt")(
3914
+ function* (now: DateTime.Utc) {
3915
+ const nowIso = DateTime.formatIso(now);
3916
+ const rows = yield* sql<{
3917
+ readonly reset_ms: number | null;
3918
+ readonly snooze_ms: number | null;
3919
+ }>`
3920
+ SELECT
3921
+ MIN(CASE WHEN julianday(reset_at) > julianday(${nowIso})
3922
+ THEN (julianday(reset_at) - 2440587.5) * 86400000 END) AS reset_ms,
3923
+ MIN(CASE WHEN julianday(snoozed_until) > julianday(${nowIso})
3924
+ THEN (julianday(snoozed_until) - 2440587.5) * 86400000 END) AS snooze_ms
3925
+ FROM (
3926
+ SELECT json_extract(payload_json, '$.limitRecovery.resetAt') AS reset_at,
3927
+ json_extract(payload_json, '$.snoozedUntil') AS snoozed_until
3928
+ FROM orchestration_v2_projection_threads
3929
+ WHERE deleted_at IS NULL
3930
+ AND json_extract(payload_json, '$.limitRecovery.autoResume') = 1
3931
+ )
3932
+ `;
3933
+ const wakes = [rows[0]?.reset_ms, rows[0]?.snooze_ms].filter(
3934
+ (value): value is number => value != null,
3935
+ );
3936
+ return wakes.length === 0 ? null : Math.round(Math.min(...wakes));
3937
+ },
3938
+ Effect.mapError((cause) => new ProjectionStoreSetupError({ cause })),
3939
+ );
3940
+
3941
const getRecoveryThreadIds = Effect.fn("ProjectionStore.getRecoveryThreadIds")(
3942
function* (kind: ProjectionRecoveryKind) {
3943
const candidates = (() => {
@@ -5204,6 +5279,13 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
5279
Effect.map((rows) => rows[0]?.count ?? 0),
5280
Effect.mapError(controlReadError(threadId)),
5281
);
5282
+ const getSubagentCount: ProjectionStoreV2Shape["getSubagentCount"] = (threadId) =>
5283
+ sql<{
5284
+ count: number;
5285
+ }>`SELECT COUNT(*) AS count FROM orchestration_v2_projection_subagents WHERE thread_id = ${threadId}`.pipe(
5286
+ Effect.map((rows) => rows[0]?.count ?? 0),
5287
+ Effect.mapError(controlReadError(threadId)),
5288
+ );
5289
const getNextTurnItemOrdinal: ProjectionStoreV2Shape["getNextTurnItemOrdinal"] = (threadId) =>
5290
sql<{ ordinal: number | null }>`SELECT MAX(ordinal) AS ordinal
5291
FROM orchestration_v2_projection_turn_items WHERE thread_id = ${threadId}`.pipe(
@@ -6918,6 +7000,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
7000
hasUnpairedRunInterruptRequest: readOnReader(hasUnpairedRunInterruptRequest),
7001
isNodeStored: readOnReader(isNodeStored),
7002
getMessageCount: readOnReader(getMessageCount),
7003
+ getSubagentCount: readOnReader(getSubagentCount),
7004
getNextTurnItemOrdinal: readOnReader(getNextTurnItemOrdinal),
7005
getTurnItem: readOnReader(getTurnItem),
7006
getThreadRecords: (threadId, fields, filter) =>
@@ -6926,6 +7009,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
7009
getPlan: readOnReader(getPlan),
7010
getProviderControlContext: readOnReader(getProviderControlContext),
7011
getLimitRecoveryCandidates: readOnReader(getLimitRecoveryCandidates),
7012
+ getLimitRecoveryWakeAt: readOnReader(getLimitRecoveryWakeAt),
7013
getRecoveryThreadIds: readOnReader(getRecoveryThreadIds),
7014
getUnreadableThreadIds: readOnReader(getUnreadableThreadIds),
7015
getThreadSnapshot: readOnReader(getThreadSnapshot),
apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts
+2
@@ -210,6 +210,7 @@ it.effect(
210
ProjectionStore.ProjectionStoreV2.of({
211
apply: () => Effect.void,
212
getLimitRecoveryCandidates: () => Effect.die("unused getLimitRecoveryCandidates"),
213
+ getLimitRecoveryWakeAt: () => Effect.die("unused getLimitRecoveryWakeAt"),
214
getOpenRemoteDelegatedTasks: Effect.die("unused getOpenRemoteDelegatedTasks"),
215
getShellSnapshot: () => Effect.die("unused getShellSnapshot"),
216
listActiveSubagentThreadIds: () => Effect.die("unused listActiveSubagentThreadIds"),
@@ -235,6 +236,7 @@ it.effect(
236
getThreadHistoryPage: () => Effect.die("unused"),
237
getTimelinePage: () => Effect.die("Unused timeline read"),
238
getMessageCount: () => Effect.die("unused message count"),
239
+ getSubagentCount: () => Effect.die("unused subagent count"),
240
getNextTurnItemOrdinal: () => Effect.die("unused ordinal read"),
241
getTurnItem: () => Effect.die("unused turn item read"),
242
getThreadRecords: () => Effect.die("unused record read"),
apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts
+1
@@ -222,6 +222,7 @@ const makeHarness = Effect.fn("makePullRequestSyncHarness")(function* (options:
222
223
const layerDependencies = Layer.mergeAll(
224
Layer.mock(PullRequestService.PullRequestService)({
225
+ withSharedProjectReads: (effect) => effect,
226
summary,
227
stack,
228
invalidate: options.invalidate ?? (() => Effect.void),
apps/server/src/orchestration-v2/PullRequestSyncReactor.ts
+1
-1
@@ -599,7 +599,7 @@ export const make = Effect.gen(function* () {
599
const worker = yield* makeDrainableWorker((scope: "all" | "requested") =>
600
Effect.suspend(() => {
601
if (scope === "requested") requestedSweepQueued = false;
602
- return sweep(scope);
602
+ return pullRequests.withSharedProjectReads(sweep(scope));
603
}).pipe(Effect.catchCause(logSkipped("pull request sync sweep failed", {}))),
604
);
605
apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts
new
+70
@@ -0,0 +1,70 @@
1
+import { assert, it } from "@effect/vitest";
2
+import type { OrchestrationV2DomainEvent } from "@t3tools/contracts";
3
+import * as Clock from "effect/Clock";
4
+import * as DateTime from "effect/DateTime";
5
+import * as Effect from "effect/Effect";
6
+import * as Layer from "effect/Layer";
7
+import * as Queue from "effect/Queue";
8
+import * as Stream from "effect/Stream";
9
+import * as TestClock from "effect/testing/TestClock";
10
+
11
+import * as Scheduler from "../scheduling/Scheduler.ts";
12
+import * as ServerSettings from "../serverSettings.ts";
13
+import * as Orchestrator from "./Orchestrator.ts";
14
+import * as ProjectionStore from "./ProjectionStore.ts";
15
+import * as ThreadManagement from "./ThreadManagementService.ts";
16
+import * as UsageLimitRecoveryWorker from "./UsageLimitRecoveryWorker.ts";
17
+
18
+it.effect("idle sweeps skip SQL until the next wake time or a relevant event", () =>
19
+ Effect.gen(function* () {
20
+ yield* TestClock.setTime(Date.parse("2026-07-01T09:00:00.000Z"));
21
+ const startMs = Date.parse("2026-07-01T09:00:00.000Z");
22
+ const queries = yield* Queue.unbounded<number>();
23
+ const events = yield* Queue.unbounded<OrchestrationV2DomainEvent>();
24
+ const layer = UsageLimitRecoveryWorker.layer.pipe(
25
+ Layer.provide(
26
+ Layer.mergeAll(
27
+ Scheduler.layer,
28
+ Layer.mock(ProjectionStore.ProjectionStoreV2)({
29
+ getLimitRecoveryCandidates: () =>
30
+ Clock.currentTimeMillis.pipe(
31
+ Effect.flatMap((nowMs) => Queue.offer(queries, nowMs - startMs)),
32
+ Effect.as([]),
33
+ ),
34
+ // One armed thread resets at 60s.
35
+ getLimitRecoveryWakeAt: (now) =>
36
+ Effect.succeed(
37
+ DateTime.toEpochMillis(now) < startMs + 60_000 ? startMs + 60_000 : null,
38
+ ),
39
+ }),
40
+ Layer.mock(ThreadManagement.ThreadManagementService)({}),
41
+ Layer.mock(Orchestrator.OrchestratorV2)({
42
+ streamDomainEvents: Stream.fromQueue(events),
43
+ }),
44
+ Layer.mock(ServerSettings.ServerSettingsService)({
45
+ getSettings: Effect.succeed({
46
+ autoResumeLimitedThreads: true,
47
+ snoozeLimitedThreads: false,
48
+ } as never),
49
+ streamChanges: Stream.never,
50
+ }),
51
+ ),
52
+ ),
53
+ );
54
+ yield* Effect.gen(function* () {
55
+ assert.equal(yield* Queue.take(queries), 0);
56
+ // Eleven idle ticks before the recorded wake time, then the wake tick itself.
57
+ yield* TestClock.adjust("60 seconds");
58
+ assert.equal(yield* Queue.take(queries), 60_000);
59
+ yield* Queue.offer(events, {
60
+ type: "turn-item.updated",
61
+ payload: { type: "assistant_message" },
62
+ } as never);
63
+ yield* TestClock.adjust("30 seconds");
64
+ yield* Queue.offer(events, { type: "run.updated", payload: { status: "failed" } } as never);
65
+ yield* TestClock.adjust("5 seconds");
66
+ // A streaming item at 90s does not wake it; the failed run does on the 95s tick.
67
+ assert.equal(yield* Queue.take(queries), 95_000);
68
+ }).pipe(Effect.provide(layer));
69
+ }),
70
+);
apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts
+30
-2
@@ -2,7 +2,11 @@ import { CommandId, MessageId, type OrchestrationV2Command } from "@t3tools/cont
2
import * as DateTime from "effect/DateTime";
3
import * as Effect from "effect/Effect";
4
import * as Layer from "effect/Layer";
5
+import * as Stream from "effect/Stream";
6
+import type { OrchestrationV2DomainEvent } from "@t3tools/contracts";
7
+import * as DueHint from "../scheduling/DueHint.ts";
8
import * as Scheduler from "../scheduling/Scheduler.ts";
9
+import { forkParked } from "../serverActivation.ts";
10
import * as ServerSettings from "../serverSettings.ts";
11
import * as ProjectionStore from "./ProjectionStore.ts";
12
import * as Orchestrator from "./Orchestrator.ts";
@@ -71,20 +75,39 @@ export function limitRecoveryCommand(
75
};
76
}
77
78
+// Events that can make a thread a candidate or change its command; streaming items cannot.
79
+const affectsLimitRecovery = (event: OrchestrationV2DomainEvent) =>
80
+ event.type.startsWith("thread.") ||
81
+ event.type === "run.created" ||
82
+ event.type === "run.updated" ||
83
+ event.type === "runtime-request.updated" ||
84
+ event.type.startsWith("provider-session.") ||
85
+ (event.type === "turn-item.updated" && event.payload.type === "error");
86
+
87
const makeSweep = Effect.gen(function* () {
88
const projections = yield* ProjectionStore.ProjectionStoreV2;
89
const threads = yield* ThreadManagement.ThreadManagementService;
90
const orchestrator = yield* Orchestrator.OrchestratorV2;
91
const settings = yield* ServerSettings.ServerSettingsService;
92
+ const due = DueHint.make();
93
+ yield* forkParked(
94
+ Stream.runForEach(orchestrator.streamDomainEvents, (event) =>
95
+ affectsLimitRecovery(event) ? due.invalidate : Effect.void,
96
+ ),
97
+ );
98
+ yield* forkParked(Stream.runForEach(settings.streamChanges, () => due.invalidate));
99
return Effect.fn("UsageLimitRecoveryWorker.sweep")(function* () {
80
- const preferences = yield* settings.getSettings;
100
const now = yield* DateTime.now;
101
+ const nowMs = DateTime.toEpochMillis(now);
102
+ if (yield* due.isIdle(nowMs)) return;
103
+ const readVersion = yield* due.version;
104
+ const preferences = yield* settings.getSettings;
105
const candidates = yield* projections.getLimitRecoveryCandidates({
106
now,
107
autoResume: preferences.autoResumeLimitedThreads,
108
snooze: preferences.snoozeLimitedThreads,
109
});
87
- const nowMs = DateTime.toEpochMillis(now);
110
+ let acted = false;
111
for (const thread of candidates) {
112
const command = limitRecoveryCommand(
113
thread,
@@ -93,6 +116,7 @@ const makeSweep = Effect.gen(function* () {
116
preferences.snoozeLimitedThreads,
117
);
118
if (command === null) continue;
119
+ acted = true;
120
yield* Effect.gen(function* () {
121
if (command.type === "thread.metadata.update" && command.limitRecovery != null) {
122
yield* orchestrator.autoArmLimitRecovery({
@@ -112,6 +136,10 @@ const makeSweep = Effect.gen(function* () {
136
),
137
);
138
}
139
+ // A sweep that acted, or failed to, re-checks on the next tick.
140
+ if (acted) return;
141
+ const wakeAt = yield* projections.getLimitRecoveryWakeAt(now);
142
+ yield* due.record(readVersion, wakeAt ?? Infinity, nowMs);
143
});
144
});
145
apps/server/src/persistence/Migrations.ts
+2
@@ -77,6 +77,7 @@ import Migration0071 from "./Migrations/071_PeerLinkFence.ts";
77
import Migration0072 from "./Migrations/072_WaitingLinkRequestIndex.ts";
78
import Migration0073 from "./Migrations/073_ActiveTurnItemIndexes.ts";
79
import Migration0074 from "./Migrations/074_ExcludeAttachmentPlaceholdersFromSearch.ts";
80
+import Migration0075 from "./Migrations/075_SubagentDeliveryState.ts";
81
82
const migrationEntries = [
83
[1, "OrchestrationEvents", Migration0001],
@@ -154,6 +155,7 @@ const migrationEntries = [
155
[72, "WaitingLinkRequestIndex", Migration0072],
156
[73, "ActiveTurnItemIndexes", Migration0073],
157
[74, "ExcludeAttachmentPlaceholdersFromSearch", Migration0074],
158
+ [75, "SubagentDeliveryState", Migration0075],
159
] as const;
160
161
export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const);
apps/server/src/persistence/Migrations/075_SubagentDeliveryState.ts
new
+20
@@ -0,0 +1,20 @@
1
+import * as Effect from "effect/Effect";
2
+import * as SqlClient from "effect/sql/SqlClient";
3
+
4
+export default Effect.gen(function* () {
5
+ const sql = yield* SqlClient.SqlClient;
6
+
7
+ // Command reads skip settled subagents by delivery state. Read from payload_json, that parsed
8
+ // every stored prompt; an added column sits after the payload's overflow pages, so the index
9
+ // must cover the whole filter.
10
+ yield* sql`ALTER TABLE orchestration_v2_projection_subagents ADD COLUMN completion_delivery_state TEXT`;
11
+ yield* sql`
12
+ UPDATE orchestration_v2_projection_subagents
13
+ SET completion_delivery_state = json_extract(payload_json, '$.completionDelivery.state')
14
+ WHERE json_type(payload_json, '$.completionDelivery') IS NOT NULL
15
+ `;
16
+ yield* sql`
17
+ CREATE INDEX orchestration_v2_projection_subagents_settled_idx
18
+ ON orchestration_v2_projection_subagents(thread_id, status, completion_delivery_state, run_id)
19
+ `;
20
+});
apps/server/src/persistence/reconcileV2PreviewMigration.test.ts
+2
@@ -54,6 +54,7 @@ describe("V2 preview upgrade", () => {
54
[72, "WaitingLinkRequestIndex"],
55
[73, "ActiveTurnItemIndexes"],
56
[74, "ExcludeAttachmentPlaceholdersFromSearch"],
57
+ [75, "SubagentDeliveryState"],
58
]);
59
assert.deepStrictEqual(yield* runMigrations(), []);
60
assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports);
@@ -151,6 +152,7 @@ describe("V2 preview upgrade", () => {
152
[72, "WaitingLinkRequestIndex"],
153
[73, "ActiveTurnItemIndexes"],
154
[74, "ExcludeAttachmentPlaceholdersFromSearch"],
155
+ [75, "SubagentDeliveryState"],
156
]);
157
}).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))),
158
);
apps/server/src/pullRequest/PullRequestService.test.ts
+29
-4
@@ -257,6 +257,7 @@ function makeService(input: {
257
readonly providers: ReadonlyArray<PullRequestProviderApi>;
258
readonly resolveHandle?: SourceControlProviderRegistry.SourceControlProviderRegistry["Service"]["resolveHandle"];
259
readonly resolveRepositoryIdentity?: RepositoryIdentityResolver.RepositoryIdentityResolver["Service"]["resolve"];
260
+ readonly onProjectRead?: () => void;
261
}) {
262
// Build in the test scope; per-call Effect.provide would close the marks database on service acquisition.
263
return Effect.flatMap(
@@ -280,11 +281,17 @@ function makeService(input: {
281
}),
282
Layer.mock(ProjectService.ProjectService)({
283
listShells: (options) =>
283
- Effect.succeed(
284
- input.projects.filter((project) => options?.projectIds?.includes(project.id) ?? true),
285
- ),
284
+ Effect.sync(() => {
285
+ input.onProjectRead?.();
286
+ return input.projects.filter(
287
+ (project) => options?.projectIds?.includes(project.id) ?? true,
288
+ );
289
+ }),
290
getShell: (projectId) =>
287
- Effect.succeed(Option.fromNullishOr(input.projects.find((p) => p.id === projectId))),
291
+ Effect.sync(() => {
292
+ input.onProjectRead?.();
293
+ return Option.fromNullishOr(input.projects.find((p) => p.id === projectId));
294
+ }),
295
}),
296
Layer.mock(RepositoryIdentityResolver.RepositoryIdentityResolver)({
297
resolve: input.resolveRepositoryIdentity ?? (() => Effect.succeed(null)),
@@ -1708,7 +1715,11 @@ it.effect("resolves a project's repository identity when its shell has none cach
1715
it.effect("routes a hosted reference to another repository through a project on that host", () =>
1716
Effect.gen(function* () {
1717
const seen: Array<{ cwd: string; repository: string; host: string }> = [];
1718
+ let projectReads = 0;
1719
const service = yield* makeService({
1720
+ onProjectRead: () => {
1721
+ projectReads += 1;
1722
+ },
1723
projects: [
1724
project({ id: "frontend", title: "web", workspaceRoot: "/web", repository: "acme/web" }),
1725
],
@@ -1730,6 +1741,20 @@ it.effect("routes a hosted reference to another repository through a project on
1741
1742
assert.strictEqual(summary.number, 7);
1743
assert.deepStrictEqual(seen, [{ cwd: "/web", repository: "acme/api", host: "github.com" }]);
1744
+
1745
+ // A sync sweep resolves the project once for all of its references, then reads fresh again.
1746
+ const readsPerReference = projectReads;
1747
+ const read = (number: number) =>
1748
+ service.summary(
1749
+ { projectId: "frontend" as ProjectId, host: "github.com", repository: "acme/api", number },
1750
+ { recoverTransientFailure: false },
1751
+ );
1752
+ assert.isAbove(readsPerReference, 2);
1753
+ yield* service.withSharedProjectReads(Effect.all([read(8), read(9), read(10)]));
1754
+ // One read of the project and one of the host's projects, for all three references.
1755
+ assert.strictEqual(projectReads, readsPerReference + 2);
1756
+ yield* read(11);
1757
+ assert.strictEqual(projectReads, readsPerReference * 2 + 2);
1758
}),
1759
);
1760
apps/server/src/pullRequest/PullRequestService.ts
+39
@@ -215,6 +215,10 @@ export class PullRequestService extends Context.Service<
215
) => Effect.Effect<void, PullRequestError>;
216
readonly subscribeRefreshes: Stream.Stream<number>;
217
readonly refreshAfterTurn: (projectId: ProjectId) => Effect.Effect<void>;
218
+ /** Project and remote lookups inside `effect` read once per filter, e.g. across one sync sweep. */
219
+ readonly withSharedProjectReads: <A, E, R>(
220
+ effect: Effect.Effect<A, E, R>,
221
+ ) => Effect.Effect<A, E, R>;
222
readonly detail: (input: PullRequestRef) => Effect.Effect<PullRequestDetail, PullRequestError>;
223
readonly preview: (
224
input: PullRequestRef,
@@ -334,6 +338,12 @@ interface WorkspaceProjects {
338
readonly viewerRoots: ReadonlyMap<string, ReadonlyArray<string>>;
339
}
340
341
+// Inherited by fibers forked inside the scope, so it is closed, not just dropped, when the scope ends.
342
+const sharedProjectReads = Context.Reference<{
343
+ open: boolean;
344
+ readonly reads: Map<string, Effect.Effect<WorkspaceProjects, PullRequestError>>;
345
+} | null>("t3/PullRequestService/sharedProjectReads", { defaultValue: () => null });
346
+
347
interface RepositoryBatch {
348
readonly key: string;
349
readonly entries: ReadonlyArray<PullRequestListEntry>;
@@ -828,6 +838,34 @@ export const make = Effect.gen(function* () {
838
839
const listWorkspaceProjects = (
840
filter: Pick<PullRequestListInput, "projectId" | "projectIds" | "host">,
841
+ ): Effect.Effect<WorkspaceProjects, PullRequestError> =>
842
+ Effect.flatMap(sharedProjectReads, (shared) => {
843
+ if (shared === null || !shared.open) return readWorkspaceProjects(filter);
844
+ const key = JSON.stringify([filter.projectId, filter.projectIds, filter.host]);
845
+ const existing = shared.reads.get(key);
846
+ if (existing !== undefined) return existing;
847
+ return Effect.cached(readWorkspaceProjects(filter)).pipe(
848
+ Effect.tap((read) => Effect.sync(() => shared.reads.set(key, read))),
849
+ Effect.flatten,
850
+ );
851
+ });
852
+
853
+ const withSharedProjectReads = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
854
+ Effect.suspend(() => {
855
+ const shared = { open: true, reads: new Map() };
856
+ return effect.pipe(
857
+ Effect.provideService(sharedProjectReads, shared),
858
+ Effect.ensuring(
859
+ Effect.sync(() => {
860
+ shared.open = false;
861
+ shared.reads.clear();
862
+ }),
863
+ ),
864
+ );
865
+ });
866
+
867
+ const readWorkspaceProjects = (
868
+ filter: Pick<PullRequestListInput, "projectId" | "projectIds" | "host">,
869
): Effect.Effect<WorkspaceProjects, PullRequestError> =>
870
(filter.projectId === undefined
871
? projects.listShells(
@@ -3660,6 +3698,7 @@ export const make = Effect.gen(function* () {
3698
);
3699
3700
return PullRequestService.of({
3701
+ withSharedProjectReads,
3702
routing,
3703
routingIdentity,
3704
withRoutingCredential,
apps/server/src/scheduledTasks/ScheduledTaskService.ts
+3
-31
@@ -49,6 +49,7 @@ import * as ManagedProjectFolders from "../project/ManagedProjectFolders.ts";
49
import * as Metrics from "../observability/Metrics.ts";
50
import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts";
51
import * as SecretRequests from "../secrets/SecretRequests.ts";
52
+import * as DueHint from "../scheduling/DueHint.ts";
53
import * as Scheduler from "../scheduling/Scheduler.ts";
54
import { isMissedFixedTimeRun, isSameSchedule, nextScheduledRunAt } from "./Schedule.ts";
55
import {
@@ -396,35 +397,6 @@ export const listDueTasks = Effect.fn("ScheduledTaskService.listDueTasks")(funct
397
return tasks;
398
});
399
399
-// Backstop for a write path that forgets to invalidate; the tick still runs every few seconds.
400
-const DUE_HINT_MAX_AGE_MS = 10 * 60 * 1_000;
401
-
402
-/**
403
- * Remembers when a poller next has work so idle scheduler ticks skip SQL.
404
- * `invalidate` after every write that can move work earlier; a tick records
405
- * only if no write happened since it read `version`.
406
- */
407
-const makeDueHint = () => {
408
- let version = 0;
409
- let hint: { readonly dueAtMs: number; readonly checkedAtMs: number } | null = null;
410
- return {
411
- invalidate: Effect.sync(() => {
412
- version++;
413
- hint = null;
414
- }),
415
- version: Effect.sync(() => version),
416
- isIdle: (nowMs: number) =>
417
- Effect.sync(
418
- () =>
419
- hint !== null && hint.dueAtMs > nowMs && nowMs - hint.checkedAtMs < DUE_HINT_MAX_AGE_MS,
420
- ),
421
- record: (readVersion: number, dueAtMs: number, nowMs: number) =>
422
- Effect.sync(() => {
423
- if (readVersion === version) hint = { dueAtMs, checkedAtMs: nowMs };
424
- }),
425
- };
426
-};
427
-
400
export const layer = Layer.effect(
401
ScheduledTaskService,
402
Effect.gen(function* () {
@@ -455,8 +427,8 @@ export const layer = Layer.effect(
427
const activeRuns = yield* Ref.make<ReadonlySet<ScheduledTaskId>>(new Set());
428
// Notifications re-emit the full list; slow subscribers need only the newest dirty signal.
429
const changesPubSub = yield* PubSub.sliding<void>(1);
458
- const dueTasks = makeDueHint();
459
- const pendingWebhooks = makeDueHint();
430
+ const dueTasks = DueHint.make();
431
+ const pendingWebhooks = DueHint.make();
432
// Every scheduled_tasks write that can change next_run_at, enabled or run state notifies.
433
const notifyChanged = dueTasks.invalidate.pipe(
434
Effect.andThen(PubSub.publish(changesPubSub, undefined)),
apps/server/src/scheduling/DueHint.ts
new
+30
@@ -0,0 +1,30 @@
1
+import * as Effect from "effect/Effect";
2
+
3
+// Backstop for a write path that forgets to invalidate; the tick still runs every few seconds.
4
+const DUE_HINT_MAX_AGE_MS = 10 * 60 * 1_000;
5
+
6
+/**
7
+ * Remembers when a poller next has work so idle scheduler ticks skip SQL.
8
+ * `invalidate` after every write that can move work earlier; a tick records
9
+ * only if no write happened since it read `version`.
10
+ */
11
+export const make = () => {
12
+ let version = 0;
13
+ let hint: { readonly dueAtMs: number; readonly checkedAtMs: number } | null = null;
14
+ return {
15
+ invalidate: Effect.sync(() => {
16
+ version++;
17
+ hint = null;
18
+ }),
19
+ version: Effect.sync(() => version),
20
+ isIdle: (nowMs: number) =>
21
+ Effect.sync(
22
+ () =>
23
+ hint !== null && hint.dueAtMs > nowMs && nowMs - hint.checkedAtMs < DUE_HINT_MAX_AGE_MS,
24
+ ),
25
+ record: (readVersion: number, dueAtMs: number, nowMs: number) =>
26
+ Effect.sync(() => {
27
+ if (readVersion === version) hint = { dueAtMs, checkedAtMs: nowMs };
28
+ }),
29
+ };
30
+};