@devries-cameron20 / t3code Mirror · read-only / commits / b79d277cdd
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 09:45 UTC

fix(server): a thread_send follow-up waits only on background work its own runs started (#37793)

Julius Marminge committed Nov 16, 2026 at 15:00 UTC b79d277cddaf65858418f29d73b87fcc3d1a19c5
6 files changed +395 -13
apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts
+324
@@ -3252,6 +3252,7 @@ it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))(
3252 id: EventId.make(`event:followup-command-awaited-${ending}`),
3253 type: "provider-thread.updated",
3254 threadId: child.childThreadId,
3255 + runId: followupRun.id,
3256 driver,
3257 providerInstanceId: modelSelection.instanceId,
3258 occurredAt: now,
@@ -3432,6 +3433,329 @@ it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))(
3433 }),
3434 );
3435
3436 + it.effect("settles a follow-up whose own run started no background work", () =>
3437 + Effect.gen(function* () {
3438 + const orchestrator = yield* Orchestrator.OrchestratorV2;
3439 + const eventSink = yield* EventSink.EventSinkV2;
3440 + const now = yield* DateTime.now;
3441 + const parentThreadId = ThreadId.make("thread:followup-earlier-work-parent");
3442 + const projectId = ProjectId.make("project:followup-earlier-work");
3443 + const parentRunId = RunId.make("run:followup-earlier-work-parent");
3444 + const rootNodeId = NodeId.make("node:followup-earlier-work-parent");
3445 + yield* seedParentWithTerminalTask({
3446 + threadId: parentThreadId,
3447 + projectId,
3448 + runId: parentRunId,
3449 + rootNodeId,
3450 + taskId: NodeId.make("node:followup-earlier-work-seed"),
3451 + deliveryState: "acknowledged",
3452 + now,
3453 + });
3454 + const child = yield* seedRestartCancelledChild({
3455 + parentThreadId,
3456 + projectId,
3457 + parentRunId,
3458 + rootNodeId,
3459 + name: "followup-earlier-work-child",
3460 + completionWake: "always",
3461 + continuationPending: false,
3462 + runStatus: "completed",
3463 + now,
3464 + });
3465 + yield* orchestrator.recoverDelegatedTasks;
3466 + const parent = yield* orchestrator.getThreadProjection(parentThreadId);
3467 + const childProviderThread = {
3468 + ...parent.providerThreads[0]!,
3469 + id: ProviderThreadId.make("provider-thread:followup-earlier-work-child"),
3470 + appThreadId: child.childThreadId,
3471 + };
3472 +
3473 + // An earlier run of the child left an awaited build running.
3474 + yield* eventSink.write({
3475 + commandId: reconcileCommandId("followup-earlier-work-build"),
3476 + events: [
3477 + {
3478 + id: EventId.make("event:followup-earlier-work-build"),
3479 + type: "provider-thread.updated",
3480 + threadId: child.childThreadId,
3481 + runId: child.childRunId,
3482 + driver,
3483 + providerInstanceId: modelSelection.instanceId,
3484 + occurredAt: now,
3485 + payload: {
3486 + ...childProviderThread,
3487 + pendingBackgroundTasks: [
3488 + { taskId: "build", kind: "command" as const, awaited: true as const },
3489 + ],
3490 + },
3491 + },
3492 + ],
3493 + });
3494 +
3495 + const followupMessageId = MessageId.make("message:followup-earlier-work-send");
3496 + yield* orchestrator.dispatch({
3497 + type: "message.dispatch",
3498 + createdBy: "agent",
3499 + creationSource: "mcp",
3500 + commandId: CommandId.make("command:followup-earlier-work-send"),
3501 + threadId: child.childThreadId,
3502 + senderThreadId: parentThreadId,
3503 + messageId: followupMessageId,
3504 + text: "Reply with exactly PONG.",
3505 + attachments: [],
3506 + dispatchMode: { type: "queue_after_active" },
3507 + });
3508 + const followupRun = (yield* orchestrator.getThreadProjection(
3509 + child.childThreadId,
3510 + )).runs.find((run) => run.userMessageId === followupMessageId)!;
3511 + const followupTaskId = followupRun.delegatedTaskId!;
3512 +
3513 + // The follow-up's run answers and ends without starting any background work.
3514 + yield* eventSink.write({
3515 + commandId: reconcileCommandId("followup-earlier-work-answered"),
3516 + events: [
3517 + // The provider resends its whole roster during the follow-up's run.
3518 + {
3519 + id: EventId.make("event:followup-earlier-work-roster"),
3520 + type: "provider-thread.updated",
3521 + threadId: child.childThreadId,
3522 + runId: followupRun.id,
3523 + driver,
3524 + providerInstanceId: modelSelection.instanceId,
3525 + occurredAt: now,
3526 + payload: {
3527 + ...childProviderThread,
3528 + pendingBackgroundTasks: [
3529 + { taskId: "build", kind: "command" as const, awaited: true as const },
3530 + ],
3531 + },
3532 + },
3533 + {
3534 + id: EventId.make("event:followup-earlier-work-answer"),
3535 + type: "message.updated",
3536 + threadId: child.childThreadId,
3537 + runId: followupRun.id,
3538 + occurredAt: now,
3539 + payload: {
3540 + id: MessageId.make("message:followup-earlier-work-answer"),
3541 + threadId: child.childThreadId,
3542 + runId: followupRun.id,
3543 + nodeId: null,
3544 + role: "assistant",
3545 + text: "PONG",
3546 + attachments: [],
3547 + streaming: false,
3548 + createdBy: "agent",
3549 + creationSource: "server",
3550 + createdAt: now,
3551 + updatedAt: now,
3552 + },
3553 + },
3554 + {
3555 + id: EventId.make("event:followup-earlier-work-run-completed"),
3556 + type: "run.updated",
3557 + threadId: child.childThreadId,
3558 + runId: followupRun.id,
3559 + occurredAt: now,
3560 + payload: { ...followupRun, status: "completed", startedAt: now, completedAt: now },
3561 + },
3562 + ],
3563 + });
3564 + yield* orchestrator.recoverDelegatedTasks;
3565 +
3566 + const followup = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find(
3567 + (task) => task.id === followupTaskId,
3568 + );
3569 + assert.equal(followup?.status, "completed");
3570 + assert.equal(followup?.result, "PONG");
3571 + }),
3572 + );
3573 +
3574 + it.effect("settles a follow-up when its own work ends while another's still runs", () =>
3575 + Effect.gen(function* () {
3576 + const orchestrator = yield* Orchestrator.OrchestratorV2;
3577 + const eventSink = yield* EventSink.EventSinkV2;
3578 + const now = yield* DateTime.now;
3579 + const parentThreadId = ThreadId.make("thread:followup-own-work-parent");
3580 + const projectId = ProjectId.make("project:followup-own-work");
3581 + const parentRunId = RunId.make("run:followup-own-work-parent");
3582 + const rootNodeId = NodeId.make("node:followup-own-work-parent");
3583 + yield* seedParentWithTerminalTask({
3584 + threadId: parentThreadId,
3585 + projectId,
3586 + runId: parentRunId,
3587 + rootNodeId,
3588 + taskId: NodeId.make("node:followup-own-work-seed"),
3589 + deliveryState: "acknowledged",
3590 + now,
3591 + });
3592 + const child = yield* seedRestartCancelledChild({
3593 + parentThreadId,
3594 + projectId,
3595 + parentRunId,
3596 + rootNodeId,
3597 + name: "followup-own-work-child",
3598 + completionWake: "always",
3599 + continuationPending: false,
3600 + runStatus: "completed",
3601 + now,
3602 + });
3603 + yield* orchestrator.recoverDelegatedTasks;
3604 + const parent = yield* orchestrator.getThreadProjection(parentThreadId);
3605 + const childProviderThread = {
3606 + ...parent.providerThreads[0]!,
3607 + id: ProviderThreadId.make("provider-thread:followup-own-work-child"),
3608 + appThreadId: child.childThreadId,
3609 + };
3610 + const awaited = (taskId: string) => ({
3611 + taskId,
3612 + kind: "command" as const,
3613 + awaited: true as const,
3614 + });
3615 + const roster = (name: string, runId: RunId, tasks: ReadonlyArray<string>) => ({
3616 + id: EventId.make(`event:followup-own-work-roster-${name}`),
3617 + type: "provider-thread.updated" as const,
3618 + threadId: child.childThreadId,
3619 + runId,
3620 + driver,
3621 + providerInstanceId: modelSelection.instanceId,
3622 + occurredAt: now,
3623 + payload: { ...childProviderThread, pendingBackgroundTasks: tasks.map(awaited) },
3624 + });
3625 + const settledTask = (taskId: NodeId, afterSequence: number) =>
3626 + eventSink.stream({ afterSequence, eventType: "subagent.updated" }).pipe(
3627 + Stream.filter(
3628 + (stored) =>
3629 + stored.event.type === "subagent.updated" &&
3630 + stored.event.payload.id === taskId &&
3631 + stored.event.payload.status === "completed",
3632 + ),
3633 + Stream.take(1),
3634 + Stream.runDrain,
3635 + );
3636 + const send = (name: string) =>
3637 + Effect.gen(function* () {
3638 + const messageId = MessageId.make(`message:followup-own-work-send-${name}`);
3639 + yield* orchestrator.dispatch({
3640 + type: "message.dispatch",
3641 + createdBy: "agent",
3642 + creationSource: "mcp",
3643 + commandId: CommandId.make(`command:followup-own-work-send-${name}`),
3644 + threadId: child.childThreadId,
3645 + senderThreadId: parentThreadId,
3646 + messageId,
3647 + text: `Start build ${name} in the background and wait on it.`,
3648 + attachments: [],
3649 + dispatchMode: { type: "queue_after_active" },
3650 + });
3651 + return (yield* orchestrator.getThreadProjection(child.childThreadId)).runs.find(
3652 + (run) => run.userMessageId === messageId,
3653 + )!;
3654 + });
3655 +
3656 + // Follow-up A starts awaited build a, answers and ends its turn.
3657 + const runA = yield* send("a");
3658 + yield* eventSink.write({
3659 + commandId: reconcileCommandId("followup-own-work-a-ended"),
3660 + events: [
3661 + roster("a", runA.id, ["a"]),
3662 + {
3663 + id: EventId.make("event:followup-own-work-a-answer"),
3664 + type: "message.updated",
3665 + threadId: child.childThreadId,
3666 + runId: runA.id,
3667 + occurredAt: now,
3668 + payload: {
3669 + id: MessageId.make("message:followup-own-work-a-answer"),
3670 + threadId: child.childThreadId,
3671 + runId: runA.id,
3672 + nodeId: null,
3673 + role: "assistant",
3674 + text: "A-DONE",
3675 + attachments: [],
3676 + streaming: false,
3677 + createdBy: "agent",
3678 + creationSource: "server",
3679 + createdAt: now,
3680 + updatedAt: now,
3681 + },
3682 + },
3683 + {
3684 + id: EventId.make("event:followup-own-work-a-completed"),
3685 + type: "run.updated",
3686 + threadId: child.childThreadId,
3687 + runId: runA.id,
3688 + occurredAt: now,
3689 + payload: { ...runA, status: "completed", startedAt: now, completedAt: now },
3690 + },
3691 + ],
3692 + });
3693 + yield* orchestrator.recoverDelegatedTasks;
3694 +
3695 + // Follow-up B starts awaited build b and is still running.
3696 + const runB = yield* send("b");
3697 + yield* eventSink.write({
3698 + commandId: reconcileCommandId("followup-own-work-b-started"),
3699 + events: [
3700 + {
3701 + id: EventId.make("event:followup-own-work-b-running"),
3702 + type: "run.updated",
3703 + threadId: child.childThreadId,
3704 + runId: runB.id,
3705 + occurredAt: now,
3706 + payload: { ...runB, status: "running", startedAt: now },
3707 + },
3708 + roster("ab", runB.id, ["a", "b"]),
3709 + ],
3710 + });
3711 + // Run updates react in order, so the barrier settling means B's start was handled too.
3712 + const barrier = yield* seedRestartCancelledChild({
3713 + parentThreadId,
3714 + projectId,
3715 + parentRunId,
3716 + rootNodeId,
3717 + name: "followup-own-work-barrier",
3718 + completionWake: "always",
3719 + continuationPending: false,
3720 + runStatus: "completed",
3721 + now,
3722 + });
3723 + const beforeBarrier = yield* eventSink.latestSequence();
3724 + yield* eventSink.write({
3725 + events: [
3726 + {
3727 + ...runEvent({
3728 + threadId: barrier.childThreadId,
3729 + runId: barrier.childRunId,
3730 + ordinal: 1,
3731 + status: "completed",
3732 + now,
3733 + }),
3734 + id: EventId.make("event:followup-own-work-barrier-ended"),
3735 + },
3736 + ],
3737 + });
3738 + yield* settledTask(barrier.taskId, beforeBarrier);
3739 + assert.equal(
3740 + (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find(
3741 + (task) => task.id === runA.delegatedTaskId,
3742 + )?.status,
3743 + "running",
3744 + );
3745 +
3746 + // Build a ends; the roster still holds b, and no run update follows.
3747 + const beforeSettle = yield* eventSink.latestSequence();
3748 + yield* eventSink.write({
3749 + commandId: reconcileCommandId("followup-own-work-a-settled"),
3750 + events: [roster("b", runB.id, ["b"])],
3751 + });
3752 + yield* settledTask(runA.delegatedTaskId!, beforeSettle);
3753 + const tasks = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents;
3754 + assert.equal(tasks.find((task) => task.id === runA.delegatedTaskId)?.result, "A-DONE");
3755 + assert.equal(tasks.find((task) => task.id === runB.delegatedTaskId)?.status, "running");
3756 + }),
3757 + );
3758 +
3759 it.effect("waits for a run the follow-up's child queued on its own thread", () =>
3760 Effect.gen(function* () {
3761 const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests;
apps/server/src/orchestration-v2/Orchestrator.ts
+11 -7
@@ -15872,18 +15872,22 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
15872
15873 // A child whose run ends while its background tasks or its own subagents still run has no
15874 // result yet. When that work settles there is no run update to finalize it, so these events
15875 - // do it. A background roster counts once it stops holding after it was seen holding.
15876 - const providerThreadsWithBackgroundWork = new Set<string>();
15875 + // do it. A roster counts when a task seen holding leaves it, since that may be one follow-up's.
15876 + const holdingTaskIdsByProviderThread = new Map<string, ReadonlySet<string>>();
15877 const settlesChildWork = (stored: OrchestrationV2StoredEvent) => {
15878 const { event } = stored;
15879 if (event.type === "subagent.updated")
15880 return !isOrchestrationV2WorkActive(event.payload.status);
15881 if (event.type !== "provider-thread.updated") return false;
15882 - if (backgroundWorkHoldsCompletion(event.payload.pendingBackgroundTasks ?? [])) {
15883 - providerThreadsWithBackgroundWork.add(event.payload.id);
15884 - return false;
15885 - }
15886 - return providerThreadsWithBackgroundWork.delete(event.payload.id);
15882 + const holding = new Set(
15883 + (event.payload.pendingBackgroundTasks ?? [])
15884 + .filter((task) => backgroundWorkHoldsCompletion([task]))
15885 + .map((task) => task.taskId),
15886 + );
15887 + const previous = holdingTaskIdsByProviderThread.get(event.payload.id);
15888 + if (holding.size > 0) holdingTaskIdsByProviderThread.set(event.payload.id, holding);
15889 + else holdingTaskIdsByProviderThread.delete(event.payload.id);
15890 + return previous !== undefined && [...previous].some((taskId) => !holding.has(taskId));
15891 };
15892 yield* Stream.merge(
15893 eventSink.stream({
apps/server/src/orchestration-v2/ProjectionStore.ts
+48 -2
@@ -848,6 +848,26 @@ function upsertById<T extends { readonly id: string }>(items: ReadonlyArray<T>,
848 return updated;
849 }
850
851 +/** Adapters resend whole rosters without runs, so a task keeps the run it first appeared under. */
852 +function attributePendingBackgroundTasks(
853 + next: OrchestrationV2ProviderThread,
854 + previous: Pick<OrchestrationV2ProviderThread, "pendingBackgroundTasks"> | undefined,
855 + runId: OrchestrationV2DomainEvent["runId"],
856 +): OrchestrationV2ProviderThread {
857 + const tasks = next.pendingBackgroundTasks ?? [];
858 + if (tasks.every((task) => task.runId !== undefined)) return next;
859 + const known = new Map(
860 + (previous?.pendingBackgroundTasks ?? []).map((task) => [task.taskId, task.runId]),
861 + );
862 + return {
863 + ...next,
864 + pendingBackgroundTasks: tasks.map((task) => {
865 + const owner = task.runId ?? (known.has(task.taskId) ? known.get(task.taskId) : runId);
866 + return owner === undefined ? task : { ...task, runId: owner };
867 + }),
868 + };
869 +}
870 +
871 function upsertProviderTurn(
872 turns: ReadonlyArray<OrchestrationV2ProviderTurn>,
873 next: OrchestrationV2ProviderTurn,
@@ -1044,7 +1064,14 @@ export function applyToProjection(
1064 activeProviderThreadId: event.payload.id,
1065 }
1066 : base.thread,
1047 - providerThreads: upsertById(base.providerThreads, event.payload),
1067 + providerThreads: upsertById(
1068 + base.providerThreads,
1069 + attributePendingBackgroundTasks(
1070 + event.payload,
1071 + base.providerThreads.find((thread) => thread.id === event.payload.id),
1072 + event.runId,
1073 + ),
1074 + ),
1075 };
1076 case "provider-turn.updated":
1077 return {
@@ -2691,7 +2718,26 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
2718 break;
2719 }
2720 case "provider-thread.updated": {
2694 - const payloadJson = yield* encodeProviderThreadPayload(event.payload);
2721 + const existingRows = (event.payload.pendingBackgroundTasks ?? []).some(
2722 + (task) => task.runId === undefined,
2723 + )
2724 + ? yield* sql<PayloadRow>`
2725 + SELECT payload_json
2726 + FROM orchestration_v2_projection_provider_threads
2727 + WHERE provider_thread_id = ${event.payload.id}
2728 + LIMIT 1
2729 + `
2730 + : [];
2731 + const existing = existingRows[0];
2732 + const payloadJson = yield* encodeProviderThreadPayload(
2733 + attributePendingBackgroundTasks(
2734 + event.payload,
2735 + existing === undefined
2736 + ? undefined
2737 + : yield* decodeProviderThreadPayload(existing.payload_json),
2738 + event.runId,
2739 + ),
2740 + );
2741 const payload = parseEncodedPayload(payloadJson);
2742 yield* sql`
2743 INSERT INTO orchestration_v2_projection_provider_threads (
apps/server/src/orchestration-v2/RestartBackgroundNote.ts
+7 -3
@@ -227,8 +227,8 @@ export function delegatedTaskRuns(
227 }
228
229 /**
230 - * A follow-up task's runs still owe a subagent they started its result, or the child still
231 - * awaits background work, as the original task waits on both.
230 + * A follow-up task's runs still owe a subagent they started its result, or still await
231 + * background work they started, as the original task waits on both.
232 */
233 export function delegatedTaskAwaitsWork(
234 runs: ReadonlyArray<OrchestrationV2Run>,
@@ -245,7 +245,11 @@ export function delegatedTaskAwaitsWork(
245 return (
246 child.subagents.some((task) => runIds.has(task.runId) && subagentHoldsCompletion(task)) ||
247 child.providerThreads.some((thread) =>
248 - backgroundWorkHoldsCompletion(thread.pendingBackgroundTasks ?? []),
248 + backgroundWorkHoldsCompletion(
249 + (thread.pendingBackgroundTasks ?? []).filter(
250 + (task) => task.runId !== undefined && runIds.has(task.runId),
251 + ),
252 + ),
253 )
254 );
255 }
apps/server/src/orchestration-v2/testkit/fixtures/claude_background_monitor_wake/output.ts
+3 -1
@@ -39,7 +39,9 @@ export function assertClaudeBackgroundMonitorWakeOutput(
39 .filter((task) => task.taskId === MONITOR_TASK_ID);
40 // The first roster precedes task_started linking the Monitor call, so that entry reads command.
41 assert.isAbove(rosterTasks.length, 1);
42 - for (const task of rosterTasks.slice(1)) {
42 + for (const { runId, ...task } of rosterTasks.slice(1)) {
43 + // Adapter rosters carry no run; server rewrites of the stored thread carry the projection's stamp.
44 + if (runId !== undefined) assert.equal(runId, monitorItem?.runId);
45 assert.deepEqual(task, {
46 taskId: MONITOR_TASK_ID,
47 description: "Background monitor test",
packages/contracts/src/orchestrationV2.ts
+2
@@ -1060,6 +1060,8 @@ const PendingBackgroundTaskFields = {
1060 parentTaskId: Schema.optional(TrimmedNonEmptyString),
1061 /** Native id of the turn item that started this work, when it is not `taskId`. */
1062 nativeItemId: Schema.optional(TrimmedNonEmptyString),
1063 + /** The run that was executing when the work first appeared, stamped by the projection. */
1064 + runId: Schema.optional(RunId),
1065 /**
1066 * The agent declared it resumes when this work ends, so the thread holds like it does for a
1067 * subagent. Only the agent knows whether a command is a wait or a dev server it left running.