@devries-cameron20 / t3code-1 / commits / 8d54c8b1a7
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): a steer that restarts a completion wake no longer strands later subagent results (#33469)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Julius Marminge committed Nov 10, 2026 at 00:02 UTC 8d54c8b1a7ab58528ec7fcddb30fdc68a6e68ea0
2 files changed +429 -83
apps/server/src/orchestration-v2/Orchestrator.ts
+131 -83
@@ -1600,6 +1600,104 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1600 }
1601 });
1602
1603 + /** Marks a wake the provider received delivered and reserves the next one for pending siblings. */
1604 + const acceptDelegatedCompletionDelivery = Effect.fn(
1605 + "Orchestrator.acceptDelegatedCompletionDelivery",
1606 + )(function* (input: {
1607 + readonly command: OrchestrationV2ServerCommand;
1608 + readonly events: Ref.Ref<Array<OrchestrationV2DomainEvent>>;
1609 + readonly projection: Pick<
1610 + OrchestrationV2ThreadProjection,
1611 + "thread" | "runs" | "messages" | "subagents"
1612 + >;
1613 + readonly messageId: OrchestrationV2ConversationMessage["id"];
1614 + readonly now: DateTime.Utc;
1615 + }) {
1616 + const { projection, now } = input;
1617 + const message = projection.messages.find((row) => row.id === input.messageId);
1618 + const ownership = message?.delegatedCompletion;
1619 + const parentRun = projection.runs.find((run) => run.id === ownership?.parentRunId);
1620 + const cohort = parentRun?.delegatedCompletion;
1621 + const delivery = cohort?.delivery;
1622 + if (
1623 + parentRun === undefined ||
1624 + cohort === undefined ||
1625 + cohort.disposition !== "open" ||
1626 + delivery == null ||
1627 + delivery.messageId !== input.messageId ||
1628 + delivery.generation !== ownership?.generation
1629 + ) {
1630 + return undefined;
1631 + }
1632 + const threadId = projection.thread.id;
1633 + const parentIsLive = hasLiveRun({ runs: [parentRun] });
1634 + const pendingTaskIds =
1635 + projection.thread.archivedAt === null && projection.thread.deletedAt === null
1636 + ? projection.subagents
1637 + .filter(
1638 + (task) =>
1639 + task.origin === "app_owned" &&
1640 + task.runId === parentRun.id &&
1641 + task.completionDelivery?.state === "pending" &&
1642 + isTerminalDelegatedTaskStatus(task.status) &&
1643 + (!parentIsLive || task.completionWake === "always"),
1644 + )
1645 + .map((task) => task.id)
1646 + : [];
1647 + const nextDelivery =
1648 + pendingTaskIds.length === 0
1649 + ? null
1650 + : {
1651 + generation: cohort.nextGeneration,
1652 + messageId: yield* mapDelegatedCompletionError(
1653 + idAllocator.allocate.message({
1654 + threadId,
1655 + ordinal:
1656 + (yield* mapDelegatedCompletionError(projectionStore.getMessageCount(threadId))) +
1657 + 1,
1658 + }),
1659 + ),
1660 + taskIds: pendingTaskIds,
1661 + };
1662 + const emitEvent = emit(input.events, input.command);
1663 + const acceptedIds = new Set(delivery.taskIds);
1664 + const pendingIds = new Set(pendingTaskIds);
1665 + for (const task of projection.subagents) {
1666 + const state = pendingIds.has(task.id)
1667 + ? "claimed"
1668 + : acceptedIds.has(task.id) && task.completionDelivery?.state === "claimed"
1669 + ? "delivered"
1670 + : undefined;
1671 + if (state === undefined) continue;
1672 + yield* emitEvent({
1673 + type: "subagent.updated",
1674 + threadId,
1675 + runId: parentRun.id,
1676 + nodeId: task.id,
1677 + driver: task.driver,
1678 + providerInstanceId: task.providerInstanceId,
1679 + occurredAt: now,
1680 + payload: { ...task, completionDelivery: { state, observedByRunId: null }, updatedAt: now },
1681 + });
1682 + }
1683 + yield* emitEvent({
1684 + type: "run.updated",
1685 + threadId,
1686 + runId: parentRun.id,
1687 + providerInstanceId: parentRun.providerInstanceId,
1688 + occurredAt: now,
1689 + payload: {
1690 + ...parentRun,
1691 + delegatedCompletion: {
1692 + ...cohort,
1693 + nextGeneration: cohort.nextGeneration + (nextDelivery === null ? 0 : 1),
1694 + delivery: nextDelivery,
1695 + },
1696 + },
1697 + });
1698 + return parentRun.id;
1699 + });
1700 +
1701 const emitQueuedRunCancellation = Effect.fn("Orchestrator.emitQueuedRunCancellation")(
1702 function* (input: {
1703 readonly command: OrchestrationV2ServerCommand;
@@ -5698,6 +5796,15 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
5796 occurredAt: now,
5797 payload: checkpointScope,
5798 });
5799 + // The restart replaces the run's input, so the wake it was running can no longer be
5800 + // settled at its terminal. Its provider turn was running, so the wake already landed.
5801 + yield* acceptDelegatedCompletionDelivery({
5802 + command: input.command,
5803 + events: input.events,
5804 + projection: input.projection,
5805 + messageId: targetRun.userMessageId,
5806 + now,
5807 + });
5808 yield* appendSteeringMessage({
5809 runId: targetRun.id,
5810 nodeId: nextRootNodeId,
@@ -13347,95 +13454,26 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
13454 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
13455 ) {
13456 const projection = yield* getProjectionWithPendingEvents(command.threadId, events);
13350 - const message = projection.messages.find((row) => row.id === command.messageId);
13351 - const ownership = message?.delegatedCompletion;
13352 - const parentRun = projection.runs.find((run) => run.id === ownership?.parentRunId);
13353 - const cohort = parentRun?.delegatedCompletion;
13354 - const delivery = cohort?.delivery;
13457 const now = yield* DateTime.now;
13356 - const emitEvent = emit(events, command);
13357 - if (
13358 - parentRun === undefined ||
13359 - cohort === undefined ||
13360 - cohort.disposition !== "open" ||
13361 - delivery == null ||
13362 - delivery.messageId !== command.messageId ||
13363 - delivery.generation !== ownership?.generation
13364 - ) {
13365 - yield* emitEvent({
13458 + const parentRunId = yield* acceptDelegatedCompletionDelivery({
13459 + command,
13460 + events,
13461 + projection,
13462 + messageId: command.messageId,
13463 + now,
13464 + });
13465 + if (parentRunId === undefined) {
13466 + yield* emit(
13467 + events,
13468 + command,
13469 + )({
13470 type: "thread.metadata-updated",
13471 threadId: command.threadId,
13472 occurredAt: now,
13473 payload: projection.thread,
13474 });
13371 - return;
13475 }
13373 - const parentIsLive = hasLiveRun({ runs: [parentRun] });
13374 - const pendingTaskIds =
13375 - projection.thread.archivedAt === null && projection.thread.deletedAt === null
13376 - ? projection.subagents
13377 - .filter(
13378 - (task) =>
13379 - task.origin === "app_owned" &&
13380 - task.runId === parentRun.id &&
13381 - task.completionDelivery?.state === "pending" &&
13382 - isTerminalDelegatedTaskStatus(task.status) &&
13383 - (!parentIsLive || task.completionWake === "always"),
13384 - )
13385 - .map((task) => task.id)
13386 - : [];
13387 - const nextDelivery =
13388 - pendingTaskIds.length === 0
13389 - ? null
13390 - : {
13391 - generation: cohort.nextGeneration,
13392 - messageId: yield* mapDelegatedCompletionError(
13393 - idAllocator.allocate.message({
13394 - threadId: command.threadId,
13395 - ordinal:
13396 - (yield* mapDelegatedCompletionError(
13397 - projectionStore.getMessageCount(projection.thread.id),
13398 - )) + 1,
13399 - }),
13400 - ),
13401 - taskIds: pendingTaskIds,
13402 - };
13403 - const acceptedIds = new Set(delivery.taskIds);
13404 - const pendingIds = new Set(pendingTaskIds);
13405 - for (const task of projection.subagents) {
13406 - const state = pendingIds.has(task.id)
13407 - ? "claimed"
13408 - : acceptedIds.has(task.id) && task.completionDelivery?.state === "claimed"
13409 - ? "delivered"
13410 - : undefined;
13411 - if (state === undefined) continue;
13412 - yield* emitEvent({
13413 - type: "subagent.updated",
13414 - threadId: command.threadId,
13415 - runId: parentRun.id,
13416 - nodeId: task.id,
13417 - driver: task.driver,
13418 - providerInstanceId: task.providerInstanceId,
13419 - occurredAt: now,
13420 - payload: { ...task, completionDelivery: { state, observedByRunId: null }, updatedAt: now },
13421 - });
13422 - }
13423 - yield* emitEvent({
13424 - type: "run.updated",
13425 - threadId: command.threadId,
13426 - runId: parentRun.id,
13427 - providerInstanceId: parentRun.providerInstanceId,
13428 - occurredAt: now,
13429 - payload: {
13430 - ...parentRun,
13431 - delegatedCompletion: {
13432 - ...cohort,
13433 - nextGeneration: cohort.nextGeneration + (nextDelivery === null ? 0 : 1),
13434 - delivery: nextDelivery,
13435 - },
13436 - },
13437 - });
13438 - return parentRun.id;
13476 + return parentRunId;
13477 });
13478
13479 const dispatchUnsupported = (command: OrchestrationV2ServerCommand) =>
@@ -14125,7 +14163,17 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
14163 ),
14164 );
14165 }
14128 - if (command.type === "notification.delivery.accept") {
14166 + if (
14167 + command.type === "notification.delivery.accept" ||
14168 + // A steer restart that settled a wake can reserve the next one for its siblings.
14169 + ((command.type === "message.dispatch" ||
14170 + command.type === "queued-message.promote-to-steer") &&
14171 + committed.storedEvents.some(
14172 + ({ event }) =>
14173 + event.type === "subagent.updated" &&
14174 + event.payload.completionDelivery?.state === "claimed",
14175 + ))
14176 + ) {
14177 yield* mapDispatchError(command)(offerDelegatedCompletionDeliveries(command.threadId));
14178 }
14179 if (
apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts
+298
@@ -2,6 +2,7 @@ import { assert, it } from "@effect/vitest";
2 import {
3 ChatAttachmentId,
4 CommandId,
5 + EnvironmentId,
6 MessageId,
7 EventId,
8 NodeId,
@@ -19,6 +20,7 @@ import * as DateTime from "effect/DateTime";
20 import * as Deferred from "effect/Deferred";
21 import * as Effect from "effect/Effect";
22 import * as Fiber from "effect/Fiber";
23 +import * as Layer from "effect/Layer";
24 import * as Queue from "effect/Queue";
25 import * as Stream from "effect/Stream";
26 import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
@@ -29,6 +31,7 @@ import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts";
31 import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts";
32 import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace";
33 import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
34 +import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests";
35
36 const driver = ProviderDriverKind.make("codex");
37 const instanceId = ProviderInstanceId.make("codex");
@@ -1215,6 +1218,301 @@ it.effect("queues a steer until the turn ends when the provider cannot steer it"
1218 ),
1219 );
1220
1221 +// regression: #33454 a steer that restarts a completion wake run left its cohort claimed forever
1222 +const restartCompletionWake = Effect.fn("restartCompletionWake")(function* (
1223 + harness: Pick<
1224 + Effect.Success<ReturnType<typeof nextTurnSelectionHarness>>,
1225 + "events" | "started" | "startFirstTurn"
1226 + >,
1227 + siblingFinishesFirst: boolean,
1228 +) {
1229 + const orchestrator = yield* Orchestrator.OrchestratorV2;
1230 + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
1231 + const sink = yield* EventSink.EventSinkV2;
1232 + const threadId = yield* harness.startFirstTurn;
1233 + const watchRun = (runId: string, status: string) =>
1234 + orchestrator.streamDomainEvents.pipe(
1235 + Stream.filter(
1236 + (event) =>
1237 + event.type === "run.updated" &&
1238 + event.payload.id === runId &&
1239 + event.payload.status === status,
1240 + ),
1241 + Stream.take(1),
1242 + Stream.runDrain,
1243 + Effect.forkScoped,
1244 + );
1245 + const watchTurnRunning = (ordinal: number) =>
1246 + orchestrator.streamDomainEvents.pipe(
1247 + Stream.filter(
1248 + (event) =>
1249 + event.type === "provider-turn.updated" &&
1250 + event.payload.status === "running" &&
1251 + event.payload.ordinal === ordinal,
1252 + ),
1253 + Stream.take(1),
1254 + Stream.runDrain,
1255 + Effect.forkScoped,
1256 + );
1257 + const finishTurn = (turn: ProviderAdapter.ProviderAdapterV2TurnInput) =>
1258 + Effect.gen(function* () {
1259 + const providerTurn = (yield* orchestrator.getThreadProjection(threadId)).providerTurns.find(
1260 + (candidate) => candidate.runAttemptId === turn.attemptId,
1261 + )!;
1262 + const settled = yield* watchRun(turn.runId, "waiting");
1263 + yield* Queue.offer(harness.events, {
1264 + type: "provider_turn.updated",
1265 + driver,
1266 + providerTurn: {
1267 + ...providerTurn,
1268 + status: "completed",
1269 + completedAt: yield* DateTime.now,
1270 + },
1271 + });
1272 + yield* Queue.offer(harness.events, {
1273 + type: "turn.terminal",
1274 + driver,
1275 + providerThreadId: providerTurn.providerThreadId,
1276 + providerTurnId: providerTurn.id,
1277 + runOrdinal: turn.runOrdinal,
1278 + status: "completed",
1279 + failure: null,
1280 + threadDisposition: "reusable",
1281 + });
1282 + yield* Fiber.join(settled);
1283 + yield* worker.drain();
1284 + });
1285 +
1286 + const origin = harness.started[0]!;
1287 + const firstTaskId = NodeId.make("task:steered-wake-first");
1288 + const siblingTaskId = NodeId.make("task:steered-wake-sibling");
1289 + const now = yield* DateTime.now;
1290 + yield* sink.write({
1291 + events: [firstTaskId, siblingTaskId].map((taskId) => ({
1292 + id: EventId.make(`event:seed:${taskId}`),
1293 + type: "subagent.updated" as const,
1294 + threadId,
1295 + runId: origin.runId,
1296 + nodeId: taskId,
1297 + occurredAt: now,
1298 + payload: {
1299 + id: taskId,
1300 + threadId,
1301 + runId: origin.runId,
1302 + parentNodeId: origin.rootNodeId,
1303 + origin: "app_owned" as const,
1304 + createdBy: "agent" as const,
1305 + driver,
1306 + providerInstanceId: instanceId,
1307 + providerThreadId: null,
1308 + childThreadId: null,
1309 + remoteChild: {
1310 + environmentId: EnvironmentId.make("environment:box"),
1311 + threadId: ThreadId.make(`thread:remote:${taskId}`),
1312 + label: "box",
1313 + },
1314 + nativeTaskRef: null,
1315 + prompt: "Do background work",
1316 + title: null,
1317 + model: null,
1318 + completionWake: "always" as const,
1319 + completionDelivery: { state: "pending" as const, observedByRunId: null },
1320 + status: "running" as const,
1321 + result: null,
1322 + startedAt: now,
1323 + completedAt: null,
1324 + updatedAt: now,
1325 + },
1326 + })),
1327 + });
1328 + yield* finishTurn(origin);
1329 +
1330 + const completeTask = (taskId: NodeId) =>
1331 + orchestrator.dispatch({
1332 + type: "delegated_task.remote.complete",
1333 + commandId: CommandId.make(`command:remote-complete:${taskId}`),
1334 + parentThreadId: threadId,
1335 + taskId,
1336 + status: "completed",
1337 + result: `${taskId} finished`,
1338 + stoppedThere: "ran-out",
1339 + });
1340 + yield* completeTask(firstTaskId);
1341 + const firstDelivery = (yield* orchestrator.getThreadProjection(threadId)).runs.find(
1342 + (run) => run.id === origin.runId,
1343 + )!.delegatedCompletion!.delivery!;
1344 + assert.deepEqual(firstDelivery.taskIds, [firstTaskId]);
1345 + yield* orchestrator.dispatch({
1346 + type: "message.dispatch",
1347 + commandId: CommandId.make("command:first-wake"),
1348 + threadId,
1349 + messageId: firstDelivery.messageId,
1350 + text: "Delegated task completed",
1351 + attachments: [],
1352 + modelSelection: runSelection,
1353 + dispatchMode: { type: "queue_after_active" },
1354 + delegatedCompletion: {
1355 + parentRunId: origin.runId,
1356 + generation: firstDelivery.generation,
1357 + taskIds: firstDelivery.taskIds,
1358 + },
1359 + createdBy: "agent",
1360 + creationSource: "server",
1361 + });
1362 + const wakeRunning = yield* watchTurnRunning(2);
1363 + yield* orchestrator.resumeQueuedRuns;
1364 + yield* worker.drain();
1365 + yield* Fiber.join(wakeRunning);
1366 + const wake = harness.started[1]!;
1367 + assert.equal(wake.message.messageId, firstDelivery.messageId);
1368 +
1369 + if (siblingFinishesFirst) {
1370 + yield* completeTask(siblingTaskId);
1371 + }
1372 +
1373 + // The provider cannot steer natively, so the steer restarts the wake run.
1374 + const steeredRunning = yield* watchTurnRunning(3);
1375 + yield* orchestrator.dispatch({
1376 + type: "message.dispatch",
1377 + commandId: CommandId.make("command:steer-wake"),
1378 + threadId,
1379 + messageId: MessageId.make("message:steer-wake"),
1380 + text: "also check the logs",
1381 + attachments: [],
1382 + modelSelection: runSelection,
1383 + dispatchMode: { type: "steer_active", targetRunId: wake.runId },
1384 + createdBy: "user",
1385 + creationSource: "web",
1386 + });
1387 + yield* worker.drain();
1388 + yield* Fiber.join(steeredRunning);
1389 + const steered = harness.started[2]!;
1390 + assert.equal(steered.runId, wake.runId);
1391 + assert.equal(
1392 + (yield* orchestrator.getThreadProjection(threadId)).attempts.find(
1393 + (attempt) => attempt.id === steered.attemptId,
1394 + )?.reason,
1395 + "steering_restart",
1396 + );
1397 + return {
1398 + threadId,
1399 + origin,
1400 + wake,
1401 + steered,
1402 + firstTaskId,
1403 + siblingTaskId,
1404 + watchRun,
1405 + finishTurn,
1406 + completeTask,
1407 + };
1408 +});
1409 +
1410 +it.effect("wakes the parent for a sibling result after a steer restarts the completion wake", () =>
1411 + Effect.scoped(
1412 + Effect.gen(function* () {
1413 + const harness = yield* nextTurnSelectionHarness("steered-completion-wake", true, false);
1414 + yield* Effect.gen(function* () {
1415 + const orchestrator = yield* Orchestrator.OrchestratorV2;
1416 + const {
1417 + threadId,
1418 + origin,
1419 + wake,
1420 + steered,
1421 + firstTaskId,
1422 + siblingTaskId,
1423 + watchRun,
1424 + finishTurn,
1425 + completeTask,
1426 + } = yield* restartCompletionWake(harness, false);
1427 +
1428 + // A follow-up queued behind the wake starts once the terminal-run listener has settled it.
1429 + const followUp = MessageId.make("message:follow-up");
1430 + yield* orchestrator.dispatch({
1431 + type: "message.dispatch",
1432 + commandId: CommandId.make("command:follow-up"),
1433 + threadId,
1434 + messageId: followUp,
1435 + text: "follow up",
1436 + attachments: [],
1437 + modelSelection: runSelection,
1438 + dispatchMode: { type: "queue_after_active" },
1439 + createdBy: "user",
1440 + creationSource: "web",
1441 + });
1442 + const followUpRun = (yield* orchestrator.getThreadProjection(threadId)).runs.find(
1443 + (run) => run.userMessageId === followUp,
1444 + )!;
1445 + const followUpStarted = yield* watchRun(followUpRun.id, "starting");
1446 + yield* finishTurn(steered);
1447 + yield* Fiber.join(followUpStarted);
1448 + assert.equal(
1449 + (yield* orchestrator.getThreadProjection(threadId)).runs.find(
1450 + (run) => run.id === wake.runId,
1451 + )?.status,
1452 + "completed",
1453 + );
1454 +
1455 + yield* completeTask(siblingTaskId);
1456 + const final = yield* orchestrator.getThreadProjection(threadId);
1457 + const tasks = new Map(
1458 + final.subagents.map((task) => [task.id, task.completionDelivery?.state]),
1459 + );
1460 + assert.deepInclude(
1461 + final.runs.find((run) => run.id === origin.runId)?.delegatedCompletion?.delivery ?? {},
1462 + { generation: 2, taskIds: [siblingTaskId] },
1463 + "the sibling finished after the steered wake but no new completion wake was reserved",
1464 + );
1465 + assert.equal(tasks.get(siblingTaskId), "claimed");
1466 + assert.equal(
1467 + tasks.get(firstTaskId),
1468 + "delivered",
1469 + "the steered wake run's own task stays claimed after that run completed",
1470 + );
1471 + }).pipe(Effect.provide(harness.layer));
1472 + }),
1473 + ),
1474 +);
1475 +
1476 +it.effect(
1477 + "offers the next completion wake when a steer restarts a wake a sibling finished behind",
1478 + () =>
1479 + Effect.scoped(
1480 + Effect.gen(function* () {
1481 + const harness = yield* nextTurnSelectionHarness(
1482 + "steered-completion-wake-behind",
1483 + true,
1484 + false,
1485 + );
1486 + const offers: ProviderContinuationRequests.ProviderContinuationRequest[] = [];
1487 + const requests = Layer.succeed(ProviderContinuationRequests.ProviderContinuationRequests, {
1488 + offer: (request) => Effect.sync(() => void offers.push(request)),
1489 + take: Effect.never,
1490 + });
1491 + yield* Effect.gen(function* () {
1492 + const orchestrator = yield* Orchestrator.OrchestratorV2;
1493 + const { threadId, origin, firstTaskId, siblingTaskId } = yield* restartCompletionWake(
1494 + harness,
1495 + true,
1496 + );
1497 + const projection = yield* orchestrator.getThreadProjection(threadId);
1498 + const delivery = projection.runs.find((run) => run.id === origin.runId)
1499 + ?.delegatedCompletion?.delivery;
1500 + assert.deepInclude(delivery ?? {}, { generation: 2, taskIds: [siblingTaskId] });
1501 + const tasks = new Map(
1502 + projection.subagents.map((task) => [task.id, task.completionDelivery?.state]),
1503 + );
1504 + assert.equal(tasks.get(firstTaskId), "delivered");
1505 + assert.equal(tasks.get(siblingTaskId), "claimed");
1506 + assert.deepEqual(offers.at(-1)?.delegatedCompletion, {
1507 + parentRunId: origin.runId,
1508 + generation: 2,
1509 + messageId: delivery!.messageId,
1510 + });
1511 + }).pipe(Effect.provide(harness.layer.pipe(Layer.provide(requests))));
1512 + }),
1513 + ),
1514 +);
1515 +
1516 // Codex flushes held drafts after the final answer; reserve their pre-steer position before turn end.
1517 it.effect("keeps a held commentary draft before the steer when turn end flushes it", () =>
1518 Effect.scoped(