@devries-cameron20 / t3code-1 / commits / 9b1d351ba0
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

perf(server): busy threads stop decoding settled subagents and idle pollers skip work (#27711)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Julius Marminge committed Oct 30, 2026 at 12:22 UTC 9b1d351ba0f63f527f884ffb3f2e394f50388764
19 files changed +645 -92
apps/server/src/mcp/OrchestratorMcpService.ts
+48 -9
@@ -513,6 +513,21 @@ function pendingTaskApproval(
513 return approval;
514 }
515
516 +/** Every task `directAppOwnedChildTasks` can return, so callers load only those. */
517 +function directChildTaskIds(
518 + target: Pick<OrchestrationV2ThreadProjection, "thread" | "runs">,
519 + returnedRunIds: ReadonlySet<RunId | null>,
520 +): ReadonlyArray<NodeId> {
521 + const ids = new Set<NodeId>();
522 + if (target.thread.forkedFrom?.type === "node") ids.add(target.thread.forkedFrom.nodeId);
523 + for (const run of target.runs) {
524 + if (returnedRunIds.has(run.id) && run.delegatedTaskId !== undefined) {
525 + ids.add(run.delegatedTaskId);
526 + }
527 + }
528 + return [...ids];
529 +}
530 +
531 function directAppOwnedChildTasks(
532 parent: Pick<OrchestrationV2ThreadProjection, "thread" | "subagents">,
533 target: Pick<OrchestrationV2ThreadProjection, "thread" | "runs">,
@@ -1016,7 +1031,6 @@ const make = Effect.gen(function* () {
1031 [
1032 "runs",
1033 "messages",
1019 - "subagents",
1034 "providerThreads",
1035 "providerTurns",
1036 "runtimeRequests",
@@ -1034,6 +1048,15 @@ const make = Effect.gen(function* () {
1048 ),
1049 );
1050
1051 + // A dispatcher thread holds thousands of tasks, each with its full prompt; read only the ones named.
1052 + const loadTasks = (threadId: ThreadId, taskIds: ReadonlyArray<NodeId>) =>
1053 + threadManagement.getThreadRecords(threadId, ["subagents"], { subagentIds: taskIds }).pipe(
1054 + Effect.map(({ subagents }) => subagents),
1055 + Effect.mapError((error) =>
1056 + failure("orchestration_error", `Unable to read thread ${threadId}: ${errorMessage(error)}`),
1057 + ),
1058 + );
1059 +
1060 // Task waits read on every wake of a parent that may hold thousands of
1061 // tasks, so they read only the task they watch and what its status needs.
1062 const loadTaskParent = (threadId: ThreadId, taskId: NodeId) =>
@@ -2385,7 +2408,7 @@ const make = Effect.gen(function* () {
2408 const childThreadId = current.childThreadId;
2409 if (childThreadId === null) {
2410 // The task runs in a linked environment: its thread there is stopped through the link.
2388 - const { subagents } = yield* loadProjection(scope.thread.threadId);
2411 + const subagents = yield* loadTasks(scope.thread.threadId, [input.taskId]);
2412 const remoteTask = subagents.find(
2413 (task) =>
2414 task.id === input.taskId &&
@@ -2428,7 +2451,7 @@ const make = Effect.gen(function* () {
2451 }
2452 });
2453 yield* assertStoppable(childThreadId);
2431 - const parentTask = parentProjection.subagents.find(
2454 + const parentTask = (yield* loadTasks(scope.thread.threadId, [input.taskId])).find(
2455 (task) => task.id === input.taskId && task.origin === "app_owned",
2456 );
2457 const disposeCompletionDelivery =
@@ -2548,9 +2571,11 @@ const make = Effect.gen(function* () {
2571 runId: parentTask.runId,
2572 subagentId: parentTask.id,
2573 });
2551 - const { subagents } = yield* threadManagement.getThreadRecords(scope.thread.threadId, [
2552 - "subagents",
2553 - ]);
2574 + const { subagents } = yield* threadManagement.getThreadRecords(
2575 + scope.thread.threadId,
2576 + ["subagents"],
2577 + { subagentIds: [input.taskId] },
2578 + );
2579 return subagents.some((task) => task.id === input.taskId && task.status === "cancelled");
2580 }).pipe(Effect.mapError(threadManagementFailure));
2581 if (isTerminalTaskStatus(current.status)) {
@@ -2847,7 +2872,19 @@ const make = Effect.gen(function* () {
2872 page.filter((row) => row.sourceThreadId === target.thread.id).map((row) => row.item.runId),
2873 );
2874 const tasks =
2850 - parent === undefined ? [] : directAppOwnedChildTasks(parent, target, returnedRunIds);
2875 + parent === undefined
2876 + ? []
2877 + : directAppOwnedChildTasks(
2878 + {
2879 + thread: parent.thread,
2880 + subagents: yield* loadTasks(
2881 + parent.thread.id,
2882 + directChildTaskIds(target, returnedRunIds),
2883 + ),
2884 + },
2885 + target,
2886 + returnedRunIds,
2887 + );
2888 if (
2889 parent !== undefined &&
2890 scope.thread !== undefined &&
@@ -2978,14 +3015,16 @@ const make = Effect.gen(function* () {
3015 const ownedTask =
3016 result.timedOut &&
3017 status !== "waiting" &&
2981 - parent?.subagents.some(
3018 + parent !== undefined &&
3019 + taskId !== undefined &&
3020 + (yield* loadTasks(parent.thread.id, [taskId])).some(
3021 (task) =>
3022 task.id === taskId &&
3023 task.origin === "app_owned" &&
3024 task.threadId === parent.thread.id &&
3025 task.childThreadId === input.threadId &&
3026 task.completionDelivery?.state !== "disposed",
2988 - ) === true;
3027 + );
3028 // A child waiting on a question or approval needs an answer, not a wake.
3029 const childRequests = ownedTask
3030 ? (yield* threadManagement
apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts
+3 -2
@@ -4944,7 +4944,8 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
4944 });
4945
4946 // Claude has no recovered frame; the next assistant message proves recovery.
4947 - const completeProviderRetry = Effect.fn("ClaudeAdapterV2.completeProviderRetry")(function* (
4947 + // Runs on every model output frame and is almost always a no-op; only a pending retry gets a span.
4948 + const completeProviderRetry = Effect.fnUntraced(function* (
4949 context: ActiveClaudeTurnContext,
4950 updatedAt: DateTime.Utc,
4951 ) {
@@ -4977,7 +4978,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
4978 startedAt: providerRetry.startedAt,
4979 updatedAt,
4980 }),
4980 - });
4981 + }).pipe(Effect.withSpan("ClaudeAdapterV2.completeProviderRetry"));
4982 });
4983
4984 const rememberProviderThread =
apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts
+3 -2
@@ -1945,7 +1945,8 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
1945 });
1946
1947 // Only new model output proves recovery; local item/completed can arrive during upstream retries.
1948 - const completeProviderRetry = Effect.fn("CodexAdapterV2.completeProviderRetry")(function* (
1948 + // Runs on every model output frame and is almost always a no-op; only a pending retry gets a span.
1949 + const completeProviderRetry = Effect.fnUntraced(function* (
1950 context: ActiveCodexTurnContext,
1951 updatedAt: DateTime.Utc,
1952 ) {
@@ -1965,7 +1966,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
1966 type: "turn_item.updated",
1967 driver: CODEX_PROVIDER,
1968 turnItem: codexRetryTurnItem(context, providerRetry, "completed", updatedAt),
1968 - });
1969 + }).pipe(Effect.withSpan("CodexAdapterV2.completeProviderRetry"));
1970 });
1971
1972 const registerRootTurn = Effect.fn("CodexAdapterV2.registerRootTurn")(function* (input: {
apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts
+72 -13
@@ -159,6 +159,11 @@ const seedParentWithTerminalTask = (input: {
159 readonly deliveryState: "delivered" | "claimed" | "acknowledged" | "disposed";
160 readonly completionWake?: "always" | "settled_only";
161 readonly deliveryTaskIds?: ReadonlyArray<NodeId>;
162 + readonly parentRunStatus?: "running" | "completed";
163 + readonly siblingTasks?: ReadonlyArray<{
164 + readonly id: NodeId;
165 + readonly deliveryState: "delivered" | "claimed";
166 + }>;
167 readonly now: DateTime.Utc;
168 }) =>
169 Effect.gen(function* () {
@@ -241,10 +246,10 @@ const seedParentWithTerminalTask = (input: {
246 userMessageId: MessageId.make(`message:seed-user:${input.threadId}`),
247 rootNodeId: input.rootNodeId,
248 activeAttemptId: null,
244 - status: "running",
249 + status: input.parentRunStatus ?? "running",
250 requestedAt: input.now,
251 startedAt: input.now,
247 - completedAt: null,
252 + completedAt: input.parentRunStatus === "completed" ? input.now : null,
253 checkpointId: null,
254 contextHandoffId: null,
255 delegatedCompletion: {
@@ -261,22 +266,25 @@ const seedParentWithTerminalTask = (input: {
266 },
267 },
268 },
264 - {
265 - id: EventId.make(`event:seed-task:${input.threadId}`),
266 - type: "subagent.updated",
269 + ...[
270 + { id: input.taskId, deliveryState: input.deliveryState },
271 + ...(input.siblingTasks ?? []),
272 + ].map((task) => ({
273 + id: EventId.make(`event:seed-task:${input.threadId}:${task.id}`),
274 + type: "subagent.updated" as const,
275 threadId: input.threadId,
276 runId: input.runId,
269 - nodeId: input.taskId,
277 + nodeId: task.id,
278 driver,
279 providerInstanceId: modelSelection.instanceId,
280 occurredAt: input.now,
281 payload: {
274 - id: input.taskId,
282 + id: task.id,
283 threadId: input.threadId,
284 runId: input.runId,
285 parentNodeId: input.rootNodeId,
278 - origin: "app_owned",
279 - createdBy: "agent",
286 + origin: "app_owned" as const,
287 + createdBy: "agent" as const,
288 driver,
289 providerInstanceId: modelSelection.instanceId,
290 providerThreadId: null,
@@ -287,16 +295,16 @@ const seedParentWithTerminalTask = (input: {
295 model: null,
296 completionWake: input.completionWake ?? "settled_only",
297 completionDelivery: {
290 - state: input.deliveryState,
291 - observedByRunId: input.deliveryState === "acknowledged" ? input.runId : null,
298 + state: task.deliveryState,
299 + observedByRunId: task.deliveryState === "acknowledged" ? input.runId : null,
300 },
293 - status: "completed",
301 + status: "completed" as const,
302 result: "child finished",
303 startedAt: input.now,
304 completedAt: input.now,
305 updatedAt: input.now,
306 },
299 - },
307 + })),
308 ],
309 });
310 });
@@ -683,6 +691,57 @@ it.layer(layerTest)("delegated completion delivery repairs", (it) => {
691 }),
692 );
693
694 + it.effect("counts already delivered siblings in the completion notice", () =>
695 + Effect.gen(function* () {
696 + const orchestrator = yield* Orchestrator.OrchestratorV2;
697 + const now = yield* DateTime.now;
698 + const threadId = ThreadId.make("thread:delegated-delivery-notice-count");
699 + const runId = RunId.make("run:delegated-delivery-notice-count");
700 + const firstTaskId = NodeId.make("node:delegated-delivery-notice-count-first");
701 + const secondTaskId = NodeId.make("node:delegated-delivery-notice-count-second");
702 +
703 + yield* seedParentWithTerminalTask({
704 + threadId,
705 + projectId: ProjectId.make("project:delegated-delivery-notice-count"),
706 + runId,
707 + rootNodeId: NodeId.make("node:delegated-delivery-notice-count-root"),
708 + taskId: firstTaskId,
709 + deliveryState: "claimed",
710 + completionWake: "always",
711 + deliveryTaskIds: [firstTaskId, secondTaskId],
712 + parentRunStatus: "completed",
713 + siblingTasks: [
714 + { id: secondTaskId, deliveryState: "claimed" },
715 + {
716 + id: NodeId.make("node:delegated-delivery-notice-count-earlier"),
717 + deliveryState: "delivered",
718 + },
719 + ],
720 + now,
721 + });
722 +
723 + yield* orchestrator.dispatch({
724 + type: "message.dispatch",
725 + commandId: CommandId.make("command:delegated-delivery-notice-count"),
726 + threadId,
727 + messageId: MessageId.make(`message:delegated-delivery:${threadId}`),
728 + text: "Delegated tasks reached a terminal state.",
729 + attachments: [],
730 + dispatchMode: { type: "queue_after_active" },
731 + createdBy: "agent",
732 + creationSource: "server",
733 + delegatedCompletion: { parentRunId: runId, generation: 1, taskIds: [firstTaskId] },
734 + });
735 +
736 + const projection = yield* orchestrator.getThreadProjection(threadId);
737 + const notice = projection.turnItems.find((item) => item.type === "notification");
738 + assert.match(
739 + notice?.type === "notification" ? notice.summary : "",
740 + /^2 of 3 delegated tasks finished:/,
741 + );
742 + }),
743 + );
744 +
745 it.effect.each([
746 {
747 name: "oversized UTF-8 result",
apps/server/src/orchestration-v2/Orchestrator.ts
+74 -24
@@ -1153,6 +1153,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1153 turnItemTypes: ["user_message", "error"],
1154 messageRoles: ["user"],
1155 omitSettledLeafNodes: true,
1156 + // Commands here read only undelivered or active tasks; a dispatcher thread keeps
1157 + // thousands of delivered ones, each carrying its full prompt.
1158 + omitSettledSubagents: true,
1159 },
1160 )
1161 .pipe(
@@ -1167,6 +1170,23 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1170 Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })),
1171 );
1172
1173 + // Command reads omit settled tasks, but a completion notice counts its whole cohort.
1174 + const notificationTasks = (
1175 + threadId: ThreadId,
1176 + message: Pick<OrchestrationV2ConversationMessage, "delegatedCompletion">,
1177 + loaded: ReadonlyArray<OrchestrationV2Subagent>,
1178 + ) =>
1179 + message.delegatedCompletion === undefined
1180 + ? Effect.succeed(loaded)
1181 + : projectionStore
1182 + .getThreadRecords(threadId, ["subagents"], {
1183 + subagentRunIds: [message.delegatedCompletion.parentRunId],
1184 + })
1185 + .pipe(
1186 + Effect.map((records) => records.subagents),
1187 + Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })),
1188 + );
1189 +
1190 const readHandoffItems = (threadId: ThreadId, runIds?: ReadonlyArray<RunId | null>) =>
1191 projectionStore
1192 .getThreadRecords(
@@ -2174,7 +2194,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
2194 nodeId: rootNodeId,
2195 providerInstanceId: queuedRun.providerInstanceId,
2196 occurredAt: now,
2177 - payload: notificationTurnItem(userTurnItem, queuedMessage, projection.subagents),
2197 + payload: notificationTurnItem(
2198 + userTurnItem,
2199 + queuedMessage,
2200 + yield* notificationTasks(threadId, queuedMessage, projection.subagents),
2201 + ),
2202 },
2203 {
2204 type: "run.updated",
@@ -4677,7 +4701,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
4701 nodeId: messageInput.nodeId,
4702 providerInstanceId: messageInput.providerInstanceId,
4703 occurredAt: now,
4680 - payload: notificationTurnItem(turnItem, message, input.projection.subagents),
4704 + payload: notificationTurnItem(
4705 + turnItem,
4706 + message,
4707 + yield* notificationTasks(input.command.threadId, message, input.projection.subagents),
4708 + ),
4709 });
4710 },
4711 );
@@ -6328,7 +6356,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
6356 nodeId: rootNodeId,
6357 providerInstanceId: modelSelection.instanceId,
6358 occurredAt: now,
6331 - payload: notificationTurnItem(turnItem, message, projection.subagents),
6359 + payload: notificationTurnItem(
6360 + turnItem,
6361 + message,
6362 + yield* notificationTasks(command.threadId, message, projection.subagents),
6363 + ),
6364 });
6365 if (preparationTurnItem !== null) {
6366 yield* emitEvent({
@@ -7222,7 +7254,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
7254 nodeId: rootNodeId,
7255 providerInstanceId: modelSelection.instanceId,
7256 occurredAt: now,
7225 - payload: notificationTurnItem(turnItem, message, projection.subagents),
7257 + payload: notificationTurnItem(
7258 + turnItem,
7259 + message,
7260 + yield* notificationTasks(command.threadId, message, projection.subagents),
7261 + ),
7262 });
7263 const forkResolution = nativeForkResolution ?? portableForkResolution;
7264 if (pendingForkTransfer !== undefined && forkResolution !== null) {
@@ -7289,14 +7325,12 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
7325 effects: Ref.Ref<Array<PendingOrchestrationEffectV2>>,
7326 ) {
7327 const parentProjection = yield* projectionStore
7292 - .getThreadRecords(command.parentThreadId, [
7293 - "runs",
7294 - "nodes",
7295 - "subagents",
7296 - "providerThreads",
7297 - "providerTurns",
7298 - "attempts",
7299 - ])
7328 + .getThreadRecords(
7329 + command.parentThreadId,
7330 + ["runs", "nodes", "providerThreads", "providerTurns", "attempts"],
7331 + // The parent node must belong to the active run, whose nodes this keeps.
7332 + { omitSettledLeafNodes: true },
7333 + )
7334 .pipe(
7335 Effect.mapError(
7336 (cause) =>
@@ -7366,7 +7400,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
7400 parentTitle: parentProjection.thread.title,
7401 prompt: command.task,
7402 ...(command.title === undefined ? {} : { title: command.title }),
7369 - ordinal: parentProjection.subagents.length + 1,
7403 + ordinal: (yield* subagentCount(command.parentThreadId)) + 1,
7404 });
7405 const childThread: OrchestrationV2AppThread = {
7406 ...makeSubagentChildThread({
@@ -7601,7 +7635,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
7635 );
7636 if (run === undefined || run.delegatedTaskId !== undefined) return;
7637 const parent = yield* projectionStore
7604 - .getThreadRecords(parentThreadId, ["runs", "subagents", "providerTurns", "attempts"])
7638 + .getThreadRecords(parentThreadId, ["runs", "subagents", "providerTurns", "attempts"], {
7639 + subagentIds: child.forkedFrom?.type === "node" ? [child.forkedFrom.nodeId] : [],
7640 + })
7641 .pipe(mapDispatchError(command));
7642 const original = parent.subagents.find(
7643 (task) =>
@@ -7729,13 +7765,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
7765 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
7766 ) {
7767 const parentProjection = yield* projectionStore
7732 - .getThreadRecords(command.parentThreadId, [
7733 - "runs",
7734 - "nodes",
7735 - "subagents",
7736 - "providerTurns",
7737 - "attempts",
7738 - ])
7768 + .getThreadRecords(command.parentThreadId, ["runs", "providerTurns", "attempts"])
7769 .pipe(
7770 Effect.mapError(
7771 (cause) => new OrchestratorProjectionError({ threadId: command.parentThreadId, cause }),
@@ -7767,7 +7797,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
7797 parentTitle: parentProjection.thread.title,
7798 prompt: command.task,
7799 ...(command.title === undefined ? {} : { title: command.title }),
7770 - ordinal: parentProjection.subagents.length + 1,
7800 + ordinal: (yield* subagentCount(command.parentThreadId)) + 1,
7801 });
7802 const common = {
7803 threadId: command.parentThreadId,
@@ -7872,11 +7902,19 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
7902 >,
7903 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
7904 ) {
7905 + // Delivery planning reads only this task and the undelivered tasks of the queued delivery;
7906 + // the task's own node and turn item are all it updates besides.
7907 const parentProjection = yield* projectionStore
7908 .getThreadRecords(
7909 command.parentThreadId,
7910 ["runs", "messages", "subagents", "providerTurns", "nodes", "turnItems"],
7879 - { turnItemTypes: ["subagent"], messageRoles: ["user"] },
7911 + {
7912 + turnItemTypes: ["subagent"],
7913 + messageRoles: ["user"],
7914 + omitSettledSubagents: { except: [command.taskId] },
7915 + nodeIds: [command.taskId],
7916 + turnItemNodeIds: [command.taskId],
7917 + },
7918 )
7919 .pipe(
7920 Effect.mapError(
@@ -8102,8 +8140,15 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
8140 command: Extract<OrchestrationV2Command, { readonly type: "delegated_task.wake-policy" }>,
8141 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
8142 ) {
8143 + // Delivery planning reads only this task and the undelivered tasks of the queued delivery.
8144 const parentProjection = yield* projectionStore
8106 - .getThreadRecords(command.parentThreadId, ["subagents", "runs", "messages", "providerTurns"])
8145 + .getThreadRecords(
8146 + command.parentThreadId,
8147 + ["subagents", "runs", "messages", "providerTurns"],
8148 + {
8149 + omitSettledSubagents: { except: [command.taskId] },
8150 + },
8151 + )
8152 .pipe(
8153 Effect.mapError(
8154 (cause) =>
@@ -9546,6 +9591,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
9591 }
9592 });
9593
9594 + const subagentCount = (threadId: ThreadId) =>
9595 + projectionStore
9596 + .getSubagentCount(threadId)
9597 + .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })));
9598 +
9599 const loadProjectionForCommand = <K extends ProjectionRecordField>(
9600 command: OrchestrationV2Command,
9601 fields: ReadonlyArray<K>,
apps/server/src/orchestration-v2/ProjectionStore.test.ts
+128
@@ -850,6 +850,121 @@ const commandReadsSkipSettledLeafNodes = Effect.gen(function* () {
850 nodeNames(yield* store.getThreadRecords(threadId, ["nodes"])),
851 nodes.map(([name]) => name).toSorted(),
852 );
853 +
854 + const subagents = [
855 + ["delivered", settledRun, "completed", "delivered"],
856 + ["acknowledged", settledRun, "failed", "acknowledged"],
857 + ["disposed", settledRun, "cancelled", "disposed"],
858 + ["pending-delivery", settledRun, "completed", "pending"],
859 + ["claimed", settledRun, "completed", "claimed"],
860 + ["no-delivery", settledRun, "interrupted", null],
861 + ["running", settledRun, "running", null],
862 + ["active-run-delivered", activeRun, "completed", "delivered"],
863 + ] as const;
864 + for (const [index, [name, run, status, delivery]] of subagents.entries()) {
865 + const id = NodeId.make(`node:settled-subagents:${name}`);
866 + yield* store.apply({
867 + id: EventId.make(`event:settled-subagents:item:${name}`),
868 + type: "turn-item.updated",
869 + threadId,
870 + runId: run.id,
871 + nodeId: id,
872 + driver,
873 + occurredAt: now,
874 + payload: {
875 + id: TurnItemId.make(`item:settled-subagents:${name}`),
876 + type: "subagent",
877 + threadId,
878 + runId: run.id,
879 + nodeId: id,
880 + providerThreadId: null,
881 + providerTurnId: null,
882 + nativeItemRef: null,
883 + parentItemId: null,
884 + ordinal: 100 + index,
885 + status: status === "running" ? "running" : "completed",
886 + title: name,
887 + startedAt: now,
888 + completedAt: status === "running" ? null : now,
889 + updatedAt: now,
890 + subagentId: id,
891 + origin: "app_owned",
892 + driver,
893 + providerInstanceId,
894 + childThreadId: null,
895 + prompt: "Do the work",
896 + result: null,
897 + },
898 + });
899 + yield* store.apply({
900 + id: EventId.make(`event:settled-subagents:${name}`),
901 + type: "subagent.updated",
902 + threadId,
903 + runId: run.id,
904 + nodeId: id,
905 + driver,
906 + providerInstanceId,
907 + occurredAt: now,
908 + payload: {
909 + id,
910 + threadId,
911 + runId: run.id,
912 + parentNodeId: run.rootNodeId!,
913 + origin: "app_owned",
914 + createdBy: "agent",
915 + driver,
916 + providerInstanceId,
917 + providerThreadId: null,
918 + childThreadId: null,
919 + nativeTaskRef: null,
920 + prompt: "Do the work",
921 + title: name,
922 + model: null,
923 + ...(delivery === null
924 + ? {}
925 + : { completionDelivery: { state: delivery, observedByRunId: null } }),
926 + status,
927 + result: null,
928 + startedAt: now,
929 + completedAt: status === "running" ? null : now,
930 + updatedAt: now,
931 + },
932 + });
933 + }
934 + const subagentNames = (records: {
935 + readonly subagents: ReadonlyArray<{ readonly title: string | null }>;
936 + }) => records.subagents.map((subagent) => subagent.title ?? "").toSorted();
937 + assert.deepEqual(
938 + subagentNames(
939 + yield* store.getThreadRecords(threadId, ["subagents"], { omitSettledSubagents: true }),
940 + ),
941 + ["active-run-delivered", "claimed", "no-delivery", "pending-delivery", "running"],
942 + );
943 + assert.deepEqual(
944 + subagentNames(yield* store.getThreadRecords(threadId, ["subagents"])),
945 + subagents.map(([name]) => name).toSorted(),
946 + );
947 + // A command about one settled task keeps it, and reads only its node and turn item.
948 + const deliveredId = NodeId.make("node:settled-subagents:delivered");
949 + const forTask = yield* store.getThreadRecords(threadId, ["subagents", "nodes", "turnItems"], {
950 + omitSettledSubagents: { except: [deliveredId] },
951 + nodeIds: [NodeId.make("node:settled-leaf-nodes:settled-tool")],
952 + turnItemNodeIds: [deliveredId],
953 + });
954 + assert.deepEqual(subagentNames(forTask), [
955 + "active-run-delivered",
956 + "claimed",
957 + "delivered",
958 + "no-delivery",
959 + "pending-delivery",
960 + "running",
961 + ]);
962 + assert.deepEqual(nodeNames(forTask), ["settled-tool"]);
963 + assert.deepEqual(
964 + forTask.turnItems.map((item) => item.title),
965 + ["delivered"],
966 + );
967 + assert.strictEqual(yield* store.getSubagentCount(threadId), subagents.length);
968 });
969
970 const backgroundThreadUpdatesKeepTimestamp = Effect.gen(function* () {
@@ -3535,6 +3650,12 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
3650 const projection = yield* projectionStore.getThreadProjection(threadId);
3651 assert.deepEqual(projection.runs[0]?.delegatedCompletion, run.delegatedCompletion);
3652 assert.deepEqual(projection.subagents[0]?.completionDelivery, task.completionDelivery);
3653 + assert.deepEqual(
3654 + (yield* projectionStore.getThreadRecords(threadId, ["subagents"], {
3655 + omitSettledSubagents: true,
3656 + })).subagents,
3657 + [],
3658 + );
3659 }),
3660 );
3661
@@ -4428,6 +4549,8 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
4549 (row) => row.id === threadId,
4550 ),
4551 );
4552 + assert.strictEqual(yield* store.getLimitRecoveryWakeAt(now), DateTime.toEpochMillis(reset));
4553 + assert.isNull(yield* store.getLimitRecoveryWakeAt(reset));
4554 const due = (yield* store.getLimitRecoveryCandidates({
4555 ...recoveryOptions,
4556 now: reset,
@@ -4450,9 +4573,14 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
4573 (row) => row.id === threadId,
4574 ),
4575 );
4576 + assert.strictEqual(
4577 + yield* store.getLimitRecoveryWakeAt(reset),
4578 + DateTime.toEpochMillis(DateTime.add(reset, { minutes: 1 })),
4579 + );
4580 yield* sql`UPDATE orchestration_v2_projection_threads
4581 SET payload_json = json_set(payload_json, '$.snoozedUntil', NULL, '$.limitRecovery.autoResume', json('false'))
4582 WHERE thread_id = ${threadId}`;
4583 + assert.isNull(yield* store.getLimitRecoveryWakeAt(now));
4584 assert.isUndefined(
4585 (yield* store.getLimitRecoveryCandidates({
4586 ...recoveryOptions,
apps/server/src/orchestration-v2/ProjectionStore.ts
+88 -4
@@ -332,11 +332,19 @@ export interface ProjectionRecordFilter {
332 readonly messageIds?: ReadonlyArray<MessageId>;
333 readonly messageRunIds?: ReadonlyArray<RunId>;
334 readonly turnItemRunIds?: ReadonlyArray<RunId | null>;
335 + readonly turnItemNodeIds?: ReadonlyArray<NodeId>;
336 readonly runIds?: ReadonlyArray<RunId>;
337 readonly subagentIds?: ReadonlyArray<NodeId>;
338 + readonly subagentRunIds?: ReadonlyArray<RunId>;
339 + readonly nodeIds?: ReadonlyArray<NodeId>;
340 readonly turnItemTypes?: ReadonlyArray<OrchestrationV2TurnItem["type"]>;
341 readonly turnItemStatuses?: ReadonlyArray<OrchestrationV2TurnItem["status"]>;
342 readonly omitSettledLeafNodes?: boolean;
343 + /**
344 + * Skips ended subagents whose completion was acknowledged, delivered or disposed, unless their
345 + * run is active or `except` names them.
346 + */
347 + readonly omitSettledSubagents?: true | { readonly except: ReadonlyArray<NodeId> };
348 }
349
350 // A fork inherits its source's history before it has messages of its own,
@@ -436,6 +444,7 @@ export interface ProjectionStoreV2Shape {
444 options: ProjectionTimelinePageOptions,
445 ) => Effect.Effect<ProjectionTimelinePage, ProjectionStoreV2Error>;
446 readonly getMessageCount: (threadId: ThreadId) => Effect.Effect<number, ProjectionStoreV2Error>;
447 + readonly getSubagentCount: (threadId: ThreadId) => Effect.Effect<number, ProjectionStoreV2Error>;
448 readonly getNextTurnItemOrdinal: (
449 threadId: ThreadId,
450 ) => Effect.Effect<number, ProjectionStoreV2Error>;
@@ -496,6 +505,10 @@ export interface ProjectionStoreV2Shape {
505 readonly autoResume: boolean;
506 readonly snooze: boolean;
507 }) => Effect.Effect<ReadonlyArray<ProjectionLimitRecoveryCandidate>, ProjectionStoreV2Error>;
508 + /** Epoch ms of the next armed reset or snooze end after `now`, when a waiting thread becomes a candidate. */
509 + readonly getLimitRecoveryWakeAt: (
510 + now: DateTime.Utc,
511 + ) => Effect.Effect<number | null, ProjectionStoreV2Error>;
512 readonly getSettlementCandidates: (
513 threadId?: ThreadId,
514 ) => Effect.Effect<ReadonlyArray<ProjectionSettlementCandidate>, ProjectionStoreV2Error>;
@@ -2107,6 +2120,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
2120 started_at,
2121 completed_at,
2122 updated_at,
2123 + completion_delivery_state,
2124 payload_json
2125 )
2126 VALUES (
@@ -2124,6 +2138,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
2138 ${nullableStringField(payload, "startedAt")},
2139 ${nullableStringField(payload, "completedAt")},
2140 ${stringField(payload, "updatedAt")},
2141 + ${event.payload.completionDelivery?.state ?? null},
2142 ${payloadJson}
2143 )
2144 ON CONFLICT(subagent_id)
@@ -2141,6 +2156,10 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
2156 started_at = excluded.started_at,
2157 completed_at = excluded.completed_at,
2158 updated_at = excluded.updated_at,
2159 + completion_delivery_state = COALESCE(
2160 + excluded.completion_delivery_state,
2161 + orchestration_v2_projection_subagents.completion_delivery_state
2162 + ),
2163 payload_json = CASE
2164 WHEN json_type(excluded.payload_json, '$.completionDelivery') IS NULL
2165 AND json_type(orchestration_v2_projection_subagents.payload_json, '$.completionDelivery') IS NOT NULL
@@ -2980,7 +2999,12 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
2999 const boundedTurnItemRows: ReadonlyArray<WindowPayloadRow> =
3000 (fields !== undefined && !fields.includes("turnItems")) ||
3001 (window === undefined &&
2983 - matchesNothing(filter?.turnItemTypes, filter?.turnItemStatuses, filter?.turnItemRunIds))
3002 + matchesNothing(
3003 + filter?.turnItemTypes,
3004 + filter?.turnItemStatuses,
3005 + filter?.turnItemRunIds,
3006 + filter?.turnItemNodeIds,
3007 + ))
3008 ? []
3009 : window === undefined
3010 ? yield* sql<PayloadRow>`
@@ -2990,6 +3014,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
3014 ${filter?.turnItemTypes === undefined ? sql`` : sql`AND type IN (SELECT value FROM json_each(${encodeIdList(filter.turnItemTypes)}))`}
3015 ${filter?.turnItemStatuses === undefined ? sql`` : sql`AND status IN (SELECT value FROM json_each(${encodeIdList(filter.turnItemStatuses)}))`}
3016 ${filter?.turnItemRunId === undefined ? sql`` : sql`AND run_id = ${filter.turnItemRunId}`}
3017 + ${filter?.turnItemNodeIds === undefined ? sql`` : sql`AND node_id IN (SELECT value FROM json_each(${encodeIdList(filter.turnItemNodeIds)}))`}
3018 ${filter?.turnItemRunIds === undefined ? sql`` : sql`AND (run_id IN (SELECT value FROM json_each(${encodeIdList(filter.turnItemRunIds.filter((id): id is RunId => id !== null))})) OR (${filter.turnItemRunIds.includes(null) ? 1 : 0} = 1 AND run_id IS NULL))`}
3019 ORDER BY ordinal ASC, turn_item_id ASC
3020 `
@@ -3216,13 +3241,15 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
3241 OR run_id IN (SELECT value FROM json_each(${cohortRunIds})))
3242 ORDER BY run_id ASC, attempt_ordinal ASC
3243 `,
3219 - fields !== undefined && !fields.includes("nodes")
3244 + (fields !== undefined && !fields.includes("nodes")) ||
3245 + (window === undefined && matchesNothing(filter?.nodeIds))
3246 ? Effect.succeed([])
3247 : window === undefined
3248 ? sql<PayloadRow>`
3249 SELECT payload_json
3250 FROM orchestration_v2_projection_nodes AS node
3251 WHERE node.thread_id = ${threadId}
3252 + ${filter?.nodeIds === undefined ? sql`` : sql`AND node.node_id IN (SELECT value FROM json_each(${encodeIdList(filter.nodeIds)}))`}
3253 ${
3254 filter?.omitSettledLeafNodes === true
3255 ? sql`AND NOT (
@@ -3253,14 +3280,34 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
3280 ),
3281 ),
3282 (fields !== undefined && !fields.includes("subagents")) ||
3256 - (window === undefined && matchesNothing(filter?.subagentIds))
3283 + (window === undefined &&
3284 + (matchesNothing(filter?.subagentIds) || matchesNothing(filter?.subagentRunIds)))
3285 ? Effect.succeed([])
3286 : window === undefined
3287 ? sql<PayloadRow>`
3288 SELECT payload_json
3261 - FROM orchestration_v2_projection_subagents
3289 + FROM orchestration_v2_projection_subagents AS subagent
3290 WHERE thread_id = ${threadId}
3291 ${filter?.subagentIds === undefined ? sql`` : sql`AND subagent_id IN (SELECT value FROM json_each(${encodeIdList(filter.subagentIds)}))`}
3292 + ${filter?.subagentRunIds === undefined ? sql`` : sql`AND run_id IN (SELECT value FROM json_each(${encodeIdList(filter.subagentRunIds)}))`}
3293 + ${
3294 + filter?.omitSettledSubagents !== undefined
3295 + ? sql`AND (${
3296 + filter.omitSettledSubagents === true
3297 + ? sql`0`
3298 + : sql`subagent_id IN (SELECT value FROM json_each(${encodeIdList(filter.omitSettledSubagents.except)}))`
3299 + } OR NOT (
3300 + subagent.status IN ('completed', 'failed', 'cancelled', 'interrupted')
3301 + AND COALESCE(subagent.completion_delivery_state, '')
3302 + IN ('acknowledged', 'delivered', 'disposed')
3303 + AND NOT EXISTS (
3304 + SELECT 1 FROM orchestration_v2_projection_runs AS run
3305 + WHERE run.run_id = subagent.run_id
3306 + AND run.status IN ('preparing', 'queued', 'starting', 'running', 'waiting')
3307 + )
3308 + ))`
3309 + : sql``
3310 + }
3311 ORDER BY COALESCE(started_at, ''), subagent_id ASC
3312 `
3313 : sql<PayloadRow>`
@@ -3863,6 +3910,34 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
3910 Effect.mapError((cause) => new ProjectionStoreSetupError({ cause })),
3911 );
3912
3913 + const getLimitRecoveryWakeAt = Effect.fn("ProjectionStore.getLimitRecoveryWakeAt")(
3914 + function* (now: DateTime.Utc) {
3915 + const nowIso = DateTime.formatIso(now);
3916 + const rows = yield* sql<{
3917 + readonly reset_ms: number | null;
3918 + readonly snooze_ms: number | null;
3919 + }>`
3920 + SELECT
3921 + MIN(CASE WHEN julianday(reset_at) > julianday(${nowIso})
3922 + THEN (julianday(reset_at) - 2440587.5) * 86400000 END) AS reset_ms,
3923 + MIN(CASE WHEN julianday(snoozed_until) > julianday(${nowIso})
3924 + THEN (julianday(snoozed_until) - 2440587.5) * 86400000 END) AS snooze_ms
3925 + FROM (
3926 + SELECT json_extract(payload_json, '$.limitRecovery.resetAt') AS reset_at,
3927 + json_extract(payload_json, '$.snoozedUntil') AS snoozed_until
3928 + FROM orchestration_v2_projection_threads
3929 + WHERE deleted_at IS NULL
3930 + AND json_extract(payload_json, '$.limitRecovery.autoResume') = 1
3931 + )
3932 + `;
3933 + const wakes = [rows[0]?.reset_ms, rows[0]?.snooze_ms].filter(
3934 + (value): value is number => value != null,
3935 + );
3936 + return wakes.length === 0 ? null : Math.round(Math.min(...wakes));
3937 + },
3938 + Effect.mapError((cause) => new ProjectionStoreSetupError({ cause })),
3939 + );
3940 +
3941 const getRecoveryThreadIds = Effect.fn("ProjectionStore.getRecoveryThreadIds")(
3942 function* (kind: ProjectionRecoveryKind) {
3943 const candidates = (() => {
@@ -5204,6 +5279,13 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
5279 Effect.map((rows) => rows[0]?.count ?? 0),
5280 Effect.mapError(controlReadError(threadId)),
5281 );
5282 + const getSubagentCount: ProjectionStoreV2Shape["getSubagentCount"] = (threadId) =>
5283 + sql<{
5284 + count: number;
5285 + }>`SELECT COUNT(*) AS count FROM orchestration_v2_projection_subagents WHERE thread_id = ${threadId}`.pipe(
5286 + Effect.map((rows) => rows[0]?.count ?? 0),
5287 + Effect.mapError(controlReadError(threadId)),
5288 + );
5289 const getNextTurnItemOrdinal: ProjectionStoreV2Shape["getNextTurnItemOrdinal"] = (threadId) =>
5290 sql<{ ordinal: number | null }>`SELECT MAX(ordinal) AS ordinal
5291 FROM orchestration_v2_projection_turn_items WHERE thread_id = ${threadId}`.pipe(
@@ -6918,6 +7000,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
7000 hasUnpairedRunInterruptRequest: readOnReader(hasUnpairedRunInterruptRequest),
7001 isNodeStored: readOnReader(isNodeStored),
7002 getMessageCount: readOnReader(getMessageCount),
7003 + getSubagentCount: readOnReader(getSubagentCount),
7004 getNextTurnItemOrdinal: readOnReader(getNextTurnItemOrdinal),
7005 getTurnItem: readOnReader(getTurnItem),
7006 getThreadRecords: (threadId, fields, filter) =>
@@ -6926,6 +7009,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
7009 getPlan: readOnReader(getPlan),
7010 getProviderControlContext: readOnReader(getProviderControlContext),
7011 getLimitRecoveryCandidates: readOnReader(getLimitRecoveryCandidates),
7012 + getLimitRecoveryWakeAt: readOnReader(getLimitRecoveryWakeAt),
7013 getRecoveryThreadIds: readOnReader(getRecoveryThreadIds),
7014 getUnreadableThreadIds: readOnReader(getUnreadableThreadIds),
7015 getThreadSnapshot: readOnReader(getThreadSnapshot),
apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts
+2
@@ -210,6 +210,7 @@ it.effect(
210 ProjectionStore.ProjectionStoreV2.of({
211 apply: () => Effect.void,
212 getLimitRecoveryCandidates: () => Effect.die("unused getLimitRecoveryCandidates"),
213 + getLimitRecoveryWakeAt: () => Effect.die("unused getLimitRecoveryWakeAt"),
214 getOpenRemoteDelegatedTasks: Effect.die("unused getOpenRemoteDelegatedTasks"),
215 getShellSnapshot: () => Effect.die("unused getShellSnapshot"),
216 listActiveSubagentThreadIds: () => Effect.die("unused listActiveSubagentThreadIds"),
@@ -235,6 +236,7 @@ it.effect(
236 getThreadHistoryPage: () => Effect.die("unused"),
237 getTimelinePage: () => Effect.die("Unused timeline read"),
238 getMessageCount: () => Effect.die("unused message count"),
239 + getSubagentCount: () => Effect.die("unused subagent count"),
240 getNextTurnItemOrdinal: () => Effect.die("unused ordinal read"),
241 getTurnItem: () => Effect.die("unused turn item read"),
242 getThreadRecords: () => Effect.die("unused record read"),
apps/server/src/orchestration-v2/PullRequestSyncReactor.test.ts
+1
@@ -222,6 +222,7 @@ const makeHarness = Effect.fn("makePullRequestSyncHarness")(function* (options:
222
223 const layerDependencies = Layer.mergeAll(
224 Layer.mock(PullRequestService.PullRequestService)({
225 + withSharedProjectReads: (effect) => effect,
226 summary,
227 stack,
228 invalidate: options.invalidate ?? (() => Effect.void),
apps/server/src/orchestration-v2/PullRequestSyncReactor.ts
+1 -1
@@ -599,7 +599,7 @@ export const make = Effect.gen(function* () {
599 const worker = yield* makeDrainableWorker((scope: "all" | "requested") =>
600 Effect.suspend(() => {
601 if (scope === "requested") requestedSweepQueued = false;
602 - return sweep(scope);
602 + return pullRequests.withSharedProjectReads(sweep(scope));
603 }).pipe(Effect.catchCause(logSkipped("pull request sync sweep failed", {}))),
604 );
605
apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.test.ts new
+70
@@ -0,0 +1,70 @@
1 +import { assert, it } from "@effect/vitest";
2 +import type { OrchestrationV2DomainEvent } from "@t3tools/contracts";
3 +import * as Clock from "effect/Clock";
4 +import * as DateTime from "effect/DateTime";
5 +import * as Effect from "effect/Effect";
6 +import * as Layer from "effect/Layer";
7 +import * as Queue from "effect/Queue";
8 +import * as Stream from "effect/Stream";
9 +import * as TestClock from "effect/testing/TestClock";
10 +
11 +import * as Scheduler from "../scheduling/Scheduler.ts";
12 +import * as ServerSettings from "../serverSettings.ts";
13 +import * as Orchestrator from "./Orchestrator.ts";
14 +import * as ProjectionStore from "./ProjectionStore.ts";
15 +import * as ThreadManagement from "./ThreadManagementService.ts";
16 +import * as UsageLimitRecoveryWorker from "./UsageLimitRecoveryWorker.ts";
17 +
18 +it.effect("idle sweeps skip SQL until the next wake time or a relevant event", () =>
19 + Effect.gen(function* () {
20 + yield* TestClock.setTime(Date.parse("2026-07-01T09:00:00.000Z"));
21 + const startMs = Date.parse("2026-07-01T09:00:00.000Z");
22 + const queries = yield* Queue.unbounded<number>();
23 + const events = yield* Queue.unbounded<OrchestrationV2DomainEvent>();
24 + const layer = UsageLimitRecoveryWorker.layer.pipe(
25 + Layer.provide(
26 + Layer.mergeAll(
27 + Scheduler.layer,
28 + Layer.mock(ProjectionStore.ProjectionStoreV2)({
29 + getLimitRecoveryCandidates: () =>
30 + Clock.currentTimeMillis.pipe(
31 + Effect.flatMap((nowMs) => Queue.offer(queries, nowMs - startMs)),
32 + Effect.as([]),
33 + ),
34 + // One armed thread resets at 60s.
35 + getLimitRecoveryWakeAt: (now) =>
36 + Effect.succeed(
37 + DateTime.toEpochMillis(now) < startMs + 60_000 ? startMs + 60_000 : null,
38 + ),
39 + }),
40 + Layer.mock(ThreadManagement.ThreadManagementService)({}),
41 + Layer.mock(Orchestrator.OrchestratorV2)({
42 + streamDomainEvents: Stream.fromQueue(events),
43 + }),
44 + Layer.mock(ServerSettings.ServerSettingsService)({
45 + getSettings: Effect.succeed({
46 + autoResumeLimitedThreads: true,
47 + snoozeLimitedThreads: false,
48 + } as never),
49 + streamChanges: Stream.never,
50 + }),
51 + ),
52 + ),
53 + );
54 + yield* Effect.gen(function* () {
55 + assert.equal(yield* Queue.take(queries), 0);
56 + // Eleven idle ticks before the recorded wake time, then the wake tick itself.
57 + yield* TestClock.adjust("60 seconds");
58 + assert.equal(yield* Queue.take(queries), 60_000);
59 + yield* Queue.offer(events, {
60 + type: "turn-item.updated",
61 + payload: { type: "assistant_message" },
62 + } as never);
63 + yield* TestClock.adjust("30 seconds");
64 + yield* Queue.offer(events, { type: "run.updated", payload: { status: "failed" } } as never);
65 + yield* TestClock.adjust("5 seconds");
66 + // A streaming item at 90s does not wake it; the failed run does on the 95s tick.
67 + assert.equal(yield* Queue.take(queries), 95_000);
68 + }).pipe(Effect.provide(layer));
69 + }),
70 +);
apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts
+30 -2
@@ -2,7 +2,11 @@ import { CommandId, MessageId, type OrchestrationV2Command } from "@t3tools/cont
2 import * as DateTime from "effect/DateTime";
3 import * as Effect from "effect/Effect";
4 import * as Layer from "effect/Layer";
5 +import * as Stream from "effect/Stream";
6 +import type { OrchestrationV2DomainEvent } from "@t3tools/contracts";
7 +import * as DueHint from "../scheduling/DueHint.ts";
8 import * as Scheduler from "../scheduling/Scheduler.ts";
9 +import { forkParked } from "../serverActivation.ts";
10 import * as ServerSettings from "../serverSettings.ts";
11 import * as ProjectionStore from "./ProjectionStore.ts";
12 import * as Orchestrator from "./Orchestrator.ts";
@@ -71,20 +75,39 @@ export function limitRecoveryCommand(
75 };
76 }
77
78 +// Events that can make a thread a candidate or change its command; streaming items cannot.
79 +const affectsLimitRecovery = (event: OrchestrationV2DomainEvent) =>
80 + event.type.startsWith("thread.") ||
81 + event.type === "run.created" ||
82 + event.type === "run.updated" ||
83 + event.type === "runtime-request.updated" ||
84 + event.type.startsWith("provider-session.") ||
85 + (event.type === "turn-item.updated" && event.payload.type === "error");
86 +
87 const makeSweep = Effect.gen(function* () {
88 const projections = yield* ProjectionStore.ProjectionStoreV2;
89 const threads = yield* ThreadManagement.ThreadManagementService;
90 const orchestrator = yield* Orchestrator.OrchestratorV2;
91 const settings = yield* ServerSettings.ServerSettingsService;
92 + const due = DueHint.make();
93 + yield* forkParked(
94 + Stream.runForEach(orchestrator.streamDomainEvents, (event) =>
95 + affectsLimitRecovery(event) ? due.invalidate : Effect.void,
96 + ),
97 + );
98 + yield* forkParked(Stream.runForEach(settings.streamChanges, () => due.invalidate));
99 return Effect.fn("UsageLimitRecoveryWorker.sweep")(function* () {
80 - const preferences = yield* settings.getSettings;
100 const now = yield* DateTime.now;
101 + const nowMs = DateTime.toEpochMillis(now);
102 + if (yield* due.isIdle(nowMs)) return;
103 + const readVersion = yield* due.version;
104 + const preferences = yield* settings.getSettings;
105 const candidates = yield* projections.getLimitRecoveryCandidates({
106 now,
107 autoResume: preferences.autoResumeLimitedThreads,
108 snooze: preferences.snoozeLimitedThreads,
109 });
87 - const nowMs = DateTime.toEpochMillis(now);
110 + let acted = false;
111 for (const thread of candidates) {
112 const command = limitRecoveryCommand(
113 thread,
@@ -93,6 +116,7 @@ const makeSweep = Effect.gen(function* () {
116 preferences.snoozeLimitedThreads,
117 );
118 if (command === null) continue;
119 + acted = true;
120 yield* Effect.gen(function* () {
121 if (command.type === "thread.metadata.update" && command.limitRecovery != null) {
122 yield* orchestrator.autoArmLimitRecovery({
@@ -112,6 +136,10 @@ const makeSweep = Effect.gen(function* () {
136 ),
137 );
138 }
139 + // A sweep that acted, or failed to, re-checks on the next tick.
140 + if (acted) return;
141 + const wakeAt = yield* projections.getLimitRecoveryWakeAt(now);
142 + yield* due.record(readVersion, wakeAt ?? Infinity, nowMs);
143 });
144 });
145
apps/server/src/persistence/Migrations.ts
+2
@@ -77,6 +77,7 @@ import Migration0071 from "./Migrations/071_PeerLinkFence.ts";
77 import Migration0072 from "./Migrations/072_WaitingLinkRequestIndex.ts";
78 import Migration0073 from "./Migrations/073_ActiveTurnItemIndexes.ts";
79 import Migration0074 from "./Migrations/074_ExcludeAttachmentPlaceholdersFromSearch.ts";
80 +import Migration0075 from "./Migrations/075_SubagentDeliveryState.ts";
81
82 const migrationEntries = [
83 [1, "OrchestrationEvents", Migration0001],
@@ -154,6 +155,7 @@ const migrationEntries = [
155 [72, "WaitingLinkRequestIndex", Migration0072],
156 [73, "ActiveTurnItemIndexes", Migration0073],
157 [74, "ExcludeAttachmentPlaceholdersFromSearch", Migration0074],
158 + [75, "SubagentDeliveryState", Migration0075],
159 ] as const;
160
161 export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const);
apps/server/src/persistence/Migrations/075_SubagentDeliveryState.ts new
+20
@@ -0,0 +1,20 @@
1 +import * as Effect from "effect/Effect";
2 +import * as SqlClient from "effect/sql/SqlClient";
3 +
4 +export default Effect.gen(function* () {
5 + const sql = yield* SqlClient.SqlClient;
6 +
7 + // Command reads skip settled subagents by delivery state. Read from payload_json, that parsed
8 + // every stored prompt; an added column sits after the payload's overflow pages, so the index
9 + // must cover the whole filter.
10 + yield* sql`ALTER TABLE orchestration_v2_projection_subagents ADD COLUMN completion_delivery_state TEXT`;
11 + yield* sql`
12 + UPDATE orchestration_v2_projection_subagents
13 + SET completion_delivery_state = json_extract(payload_json, '$.completionDelivery.state')
14 + WHERE json_type(payload_json, '$.completionDelivery') IS NOT NULL
15 + `;
16 + yield* sql`
17 + CREATE INDEX orchestration_v2_projection_subagents_settled_idx
18 + ON orchestration_v2_projection_subagents(thread_id, status, completion_delivery_state, run_id)
19 + `;
20 +});
apps/server/src/persistence/reconcileV2PreviewMigration.test.ts
+2
@@ -54,6 +54,7 @@ describe("V2 preview upgrade", () => {
54 [72, "WaitingLinkRequestIndex"],
55 [73, "ActiveTurnItemIndexes"],
56 [74, "ExcludeAttachmentPlaceholdersFromSearch"],
57 + [75, "SubagentDeliveryState"],
58 ]);
59 assert.deepStrictEqual(yield* runMigrations(), []);
60 assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports);
@@ -151,6 +152,7 @@ describe("V2 preview upgrade", () => {
152 [72, "WaitingLinkRequestIndex"],
153 [73, "ActiveTurnItemIndexes"],
154 [74, "ExcludeAttachmentPlaceholdersFromSearch"],
155 + [75, "SubagentDeliveryState"],
156 ]);
157 }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))),
158 );
apps/server/src/pullRequest/PullRequestService.test.ts
+29 -4
@@ -257,6 +257,7 @@ function makeService(input: {
257 readonly providers: ReadonlyArray<PullRequestProviderApi>;
258 readonly resolveHandle?: SourceControlProviderRegistry.SourceControlProviderRegistry["Service"]["resolveHandle"];
259 readonly resolveRepositoryIdentity?: RepositoryIdentityResolver.RepositoryIdentityResolver["Service"]["resolve"];
260 + readonly onProjectRead?: () => void;
261 }) {
262 // Build in the test scope; per-call Effect.provide would close the marks database on service acquisition.
263 return Effect.flatMap(
@@ -280,11 +281,17 @@ function makeService(input: {
281 }),
282 Layer.mock(ProjectService.ProjectService)({
283 listShells: (options) =>
283 - Effect.succeed(
284 - input.projects.filter((project) => options?.projectIds?.includes(project.id) ?? true),
285 - ),
284 + Effect.sync(() => {
285 + input.onProjectRead?.();
286 + return input.projects.filter(
287 + (project) => options?.projectIds?.includes(project.id) ?? true,
288 + );
289 + }),
290 getShell: (projectId) =>
287 - Effect.succeed(Option.fromNullishOr(input.projects.find((p) => p.id === projectId))),
291 + Effect.sync(() => {
292 + input.onProjectRead?.();
293 + return Option.fromNullishOr(input.projects.find((p) => p.id === projectId));
294 + }),
295 }),
296 Layer.mock(RepositoryIdentityResolver.RepositoryIdentityResolver)({
297 resolve: input.resolveRepositoryIdentity ?? (() => Effect.succeed(null)),
@@ -1708,7 +1715,11 @@ it.effect("resolves a project's repository identity when its shell has none cach
1715 it.effect("routes a hosted reference to another repository through a project on that host", () =>
1716 Effect.gen(function* () {
1717 const seen: Array<{ cwd: string; repository: string; host: string }> = [];
1718 + let projectReads = 0;
1719 const service = yield* makeService({
1720 + onProjectRead: () => {
1721 + projectReads += 1;
1722 + },
1723 projects: [
1724 project({ id: "frontend", title: "web", workspaceRoot: "/web", repository: "acme/web" }),
1725 ],
@@ -1730,6 +1741,20 @@ it.effect("routes a hosted reference to another repository through a project on
1741
1742 assert.strictEqual(summary.number, 7);
1743 assert.deepStrictEqual(seen, [{ cwd: "/web", repository: "acme/api", host: "github.com" }]);
1744 +
1745 + // A sync sweep resolves the project once for all of its references, then reads fresh again.
1746 + const readsPerReference = projectReads;
1747 + const read = (number: number) =>
1748 + service.summary(
1749 + { projectId: "frontend" as ProjectId, host: "github.com", repository: "acme/api", number },
1750 + { recoverTransientFailure: false },
1751 + );
1752 + assert.isAbove(readsPerReference, 2);
1753 + yield* service.withSharedProjectReads(Effect.all([read(8), read(9), read(10)]));
1754 + // One read of the project and one of the host's projects, for all three references.
1755 + assert.strictEqual(projectReads, readsPerReference + 2);
1756 + yield* read(11);
1757 + assert.strictEqual(projectReads, readsPerReference * 2 + 2);
1758 }),
1759 );
1760
apps/server/src/pullRequest/PullRequestService.ts
+39
@@ -215,6 +215,10 @@ export class PullRequestService extends Context.Service<
215 ) => Effect.Effect<void, PullRequestError>;
216 readonly subscribeRefreshes: Stream.Stream<number>;
217 readonly refreshAfterTurn: (projectId: ProjectId) => Effect.Effect<void>;
218 + /** Project and remote lookups inside `effect` read once per filter, e.g. across one sync sweep. */
219 + readonly withSharedProjectReads: <A, E, R>(
220 + effect: Effect.Effect<A, E, R>,
221 + ) => Effect.Effect<A, E, R>;
222 readonly detail: (input: PullRequestRef) => Effect.Effect<PullRequestDetail, PullRequestError>;
223 readonly preview: (
224 input: PullRequestRef,
@@ -334,6 +338,12 @@ interface WorkspaceProjects {
338 readonly viewerRoots: ReadonlyMap<string, ReadonlyArray<string>>;
339 }
340
341 +// Inherited by fibers forked inside the scope, so it is closed, not just dropped, when the scope ends.
342 +const sharedProjectReads = Context.Reference<{
343 + open: boolean;
344 + readonly reads: Map<string, Effect.Effect<WorkspaceProjects, PullRequestError>>;
345 +} | null>("t3/PullRequestService/sharedProjectReads", { defaultValue: () => null });
346 +
347 interface RepositoryBatch {
348 readonly key: string;
349 readonly entries: ReadonlyArray<PullRequestListEntry>;
@@ -828,6 +838,34 @@ export const make = Effect.gen(function* () {
838
839 const listWorkspaceProjects = (
840 filter: Pick<PullRequestListInput, "projectId" | "projectIds" | "host">,
841 + ): Effect.Effect<WorkspaceProjects, PullRequestError> =>
842 + Effect.flatMap(sharedProjectReads, (shared) => {
843 + if (shared === null || !shared.open) return readWorkspaceProjects(filter);
844 + const key = JSON.stringify([filter.projectId, filter.projectIds, filter.host]);
845 + const existing = shared.reads.get(key);
846 + if (existing !== undefined) return existing;
847 + return Effect.cached(readWorkspaceProjects(filter)).pipe(
848 + Effect.tap((read) => Effect.sync(() => shared.reads.set(key, read))),
849 + Effect.flatten,
850 + );
851 + });
852 +
853 + const withSharedProjectReads = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
854 + Effect.suspend(() => {
855 + const shared = { open: true, reads: new Map() };
856 + return effect.pipe(
857 + Effect.provideService(sharedProjectReads, shared),
858 + Effect.ensuring(
859 + Effect.sync(() => {
860 + shared.open = false;
861 + shared.reads.clear();
862 + }),
863 + ),
864 + );
865 + });
866 +
867 + const readWorkspaceProjects = (
868 + filter: Pick<PullRequestListInput, "projectId" | "projectIds" | "host">,
869 ): Effect.Effect<WorkspaceProjects, PullRequestError> =>
870 (filter.projectId === undefined
871 ? projects.listShells(
@@ -3660,6 +3698,7 @@ export const make = Effect.gen(function* () {
3698 );
3699
3700 return PullRequestService.of({
3701 + withSharedProjectReads,
3702 routing,
3703 routingIdentity,
3704 withRoutingCredential,
apps/server/src/scheduledTasks/ScheduledTaskService.ts
+3 -31
@@ -49,6 +49,7 @@ import * as ManagedProjectFolders from "../project/ManagedProjectFolders.ts";
49 import * as Metrics from "../observability/Metrics.ts";
50 import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts";
51 import * as SecretRequests from "../secrets/SecretRequests.ts";
52 +import * as DueHint from "../scheduling/DueHint.ts";
53 import * as Scheduler from "../scheduling/Scheduler.ts";
54 import { isMissedFixedTimeRun, isSameSchedule, nextScheduledRunAt } from "./Schedule.ts";
55 import {
@@ -396,35 +397,6 @@ export const listDueTasks = Effect.fn("ScheduledTaskService.listDueTasks")(funct
397 return tasks;
398 });
399
399 -// Backstop for a write path that forgets to invalidate; the tick still runs every few seconds.
400 -const DUE_HINT_MAX_AGE_MS = 10 * 60 * 1_000;
401 -
402 -/**
403 - * Remembers when a poller next has work so idle scheduler ticks skip SQL.
404 - * `invalidate` after every write that can move work earlier; a tick records
405 - * only if no write happened since it read `version`.
406 - */
407 -const makeDueHint = () => {
408 - let version = 0;
409 - let hint: { readonly dueAtMs: number; readonly checkedAtMs: number } | null = null;
410 - return {
411 - invalidate: Effect.sync(() => {
412 - version++;
413 - hint = null;
414 - }),
415 - version: Effect.sync(() => version),
416 - isIdle: (nowMs: number) =>
417 - Effect.sync(
418 - () =>
419 - hint !== null && hint.dueAtMs > nowMs && nowMs - hint.checkedAtMs < DUE_HINT_MAX_AGE_MS,
420 - ),
421 - record: (readVersion: number, dueAtMs: number, nowMs: number) =>
422 - Effect.sync(() => {
423 - if (readVersion === version) hint = { dueAtMs, checkedAtMs: nowMs };
424 - }),
425 - };
426 -};
427 -
400 export const layer = Layer.effect(
401 ScheduledTaskService,
402 Effect.gen(function* () {
@@ -455,8 +427,8 @@ export const layer = Layer.effect(
427 const activeRuns = yield* Ref.make<ReadonlySet<ScheduledTaskId>>(new Set());
428 // Notifications re-emit the full list; slow subscribers need only the newest dirty signal.
429 const changesPubSub = yield* PubSub.sliding<void>(1);
458 - const dueTasks = makeDueHint();
459 - const pendingWebhooks = makeDueHint();
430 + const dueTasks = DueHint.make();
431 + const pendingWebhooks = DueHint.make();
432 // Every scheduled_tasks write that can change next_run_at, enabled or run state notifies.
433 const notifyChanged = dueTasks.invalidate.pipe(
434 Effect.andThen(PubSub.publish(changesPubSub, undefined)),
apps/server/src/scheduling/DueHint.ts new
+30
@@ -0,0 +1,30 @@
1 +import * as Effect from "effect/Effect";
2 +
3 +// Backstop for a write path that forgets to invalidate; the tick still runs every few seconds.
4 +const DUE_HINT_MAX_AGE_MS = 10 * 60 * 1_000;
5 +
6 +/**
7 + * Remembers when a poller next has work so idle scheduler ticks skip SQL.
8 + * `invalidate` after every write that can move work earlier; a tick records
9 + * only if no write happened since it read `version`.
10 + */
11 +export const make = () => {
12 + let version = 0;
13 + let hint: { readonly dueAtMs: number; readonly checkedAtMs: number } | null = null;
14 + return {
15 + invalidate: Effect.sync(() => {
16 + version++;
17 + hint = null;
18 + }),
19 + version: Effect.sync(() => version),
20 + isIdle: (nowMs: number) =>
21 + Effect.sync(
22 + () =>
23 + hint !== null && hint.dueAtMs > nowMs && nowMs - hint.checkedAtMs < DUE_HINT_MAX_AGE_MS,
24 + ),
25 + record: (readVersion: number, dueAtMs: number, nowMs: number) =>
26 + Effect.sync(() => {
27 + if (readVersion === version) hint = { dueAtMs, checkedAtMs: nowMs };
28 + }),
29 + };
30 +};