The best way to control Claude Code, Codex and more from any computer or phone
fix(server): Stop clears a finished delegated child's row held by a stale native subagent (#32307)
Julius Marminge committed
Nov 13, 2026 at 10:37 UTC
78b654f998f651a9d7f00ecaf93499c5ed0bc5ad
2 files changed
+273
-75
apps/server/src/orchestration-v2/Orchestrator.ts
+125
-75
@@ -1450,6 +1450,21 @@ function lastDeliveredRunForProviderThread(
1450
);
1451
}
1452
1453
+type DyingNativeSubagent = {
1454
+ readonly projection: Pick<OrchestrationV2ThreadProjection, "nodes" | "subagents" | "turnItems">;
1455
+ readonly subagent: Pick<
1456
+ Extract<OrchestrationV2TurnItem, { readonly type: "subagent" }>,
1457
+ "subagentId" | "origin" | "childThreadId" | "providerInstanceId"
1458
+ >;
1459
+};
1460
+
1461
+const nativeSubagentRef = (subagent: OrchestrationV2Subagent): DyingNativeSubagent["subagent"] => ({
1462
+ subagentId: subagent.id,
1463
+ origin: subagent.origin,
1464
+ childThreadId: subagent.childThreadId,
1465
+ providerInstanceId: subagent.providerInstanceId,
1466
+});
1467
+
1468
const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(function* () {
1469
const checkpointService = yield* CheckpointServiceV2;
1470
const commandPolicy = yield* CommandPolicyV2;
@@ -11809,6 +11824,78 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
11824
}
11825
});
11826
11827
+ // Ends each native subagent's record, its own thread's runless root turns, and the
11828
+ // subagents it started there, which a provider can record on that thread rather than its parent's.
11829
+ const endNativeSubagentTrees = Effect.fn("Orchestrator.endNativeSubagentTrees")(
11830
+ function* (input: {
11831
+ readonly command: Extract<
11832
+ OrchestrationV2ServerCommand,
11833
+ { readonly type: "run.interrupt" | "thread.background-work.settle" | "thread.stop" }
11834
+ >;
11835
+ readonly events: Ref.Ref<Array<OrchestrationV2DomainEvent>>;
11836
+ readonly dying: ReadonlyArray<DyingNativeSubagent>;
11837
+ readonly childThreadIds: Set<ThreadId>;
11838
+ readonly now: DateTime.Utc;
11839
+ }) {
11840
+ const allocateEventId = () =>
11841
+ idAllocator.allocate.event({
11842
+ threadId: input.command.threadId,
11843
+ commandId: input.command.commandId,
11844
+ });
11845
+ const dying = [...input.dying];
11846
+ const { childThreadIds } = input;
11847
+ for (const { projection, subagent } of dying) {
11848
+ const ended = yield* endOrphanedNativeSubagent({
11849
+ projection,
11850
+ subagentId: subagent.subagentId,
11851
+ status: "interrupted",
11852
+ now: input.now,
11853
+ emitted: yield* Ref.get(input.events),
11854
+ allocateEventId,
11855
+ }).pipe(mapDispatchError(input.command));
11856
+ yield* Ref.update(input.events, (events) => [...events, ...ended]);
11857
+ const childThreadId = subagent.childThreadId;
11858
+ if (
11859
+ subagent.origin !== "provider_native" ||
11860
+ childThreadId === null ||
11861
+ childThreadIds.has(childThreadId)
11862
+ ) {
11863
+ continue;
11864
+ }
11865
+ childThreadIds.add(childThreadId);
11866
+ const child = yield* projectionStore
11867
+ .getThreadRecords(childThreadId, ["nodes", "providerTurns", "subagents", "turnItems"], {
11868
+ // Runless items are the subagent's; any run there is the user's own.
11869
+ turnItemRunIds: [null],
11870
+ turnItemStatuses: ["pending", "running", "waiting"],
11871
+ })
11872
+ .pipe(
11873
+ Effect.catchTags({ ProjectionStoreThreadNotFoundError: () => Effect.succeed(null) }),
11874
+ mapDispatchError(input.command),
11875
+ );
11876
+ if (child === null) continue;
11877
+ const childEnded = yield* endRunlessRootTurns({
11878
+ threadId: childThreadId,
11879
+ providerInstanceId: subagent.providerInstanceId,
11880
+ projection: child,
11881
+ status: "interrupted",
11882
+ now: input.now,
11883
+ emitted: yield* Ref.get(input.events),
11884
+ allocateEventId,
11885
+ }).pipe(mapDispatchError(input.command));
11886
+ yield* Ref.update(input.events, (events) => [...events, ...childEnded]);
11887
+ for (const nested of child.turnItems) {
11888
+ if (nested.type === "subagent") dying.push({ projection: child, subagent: nested });
11889
+ }
11890
+ for (const nested of child.subagents) {
11891
+ if (nested.origin === "provider_native" && nested.runId === null) {
11892
+ dying.push({ projection: child, subagent: nativeSubagentRef(nested) });
11893
+ }
11894
+ }
11895
+ }
11896
+ },
11897
+ );
11898
+
11899
const settleBackgroundWork = Effect.fn("Orchestrator.settleBackgroundWork")(function* (input: {
11900
readonly command: Extract<
11901
OrchestrationV2ServerCommand,
@@ -11825,11 +11912,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
11912
}) {
11913
const emitEvent = emit(input.events, input.command);
11914
const runOrdinals = new Map(input.projection.runs.map((run) => [run.id, run.ordinal]));
11828
- const allocateEventId = () =>
11829
- idAllocator.allocate.event({
11830
- threadId: input.command.threadId,
11831
- commandId: input.command.commandId,
11832
- });
11915
const childThreadIds = new Set<ThreadId>();
11916
const liveness = new Map<string, boolean>();
11917
const hasLiveSession = Effect.fn("Orchestrator.hasLiveSession")(function* (
@@ -11894,76 +11976,14 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
11976
},
11977
});
11978
if (item.type !== "subagent") continue;
11897
- // A native subagent died with the process too: end its record, its own
11898
- // thread's runless root turn, and the subagents it started there, which
11899
- // a provider can record on that thread rather than this one.
11900
- const dying: Array<{
11901
- readonly projection: Pick<
11902
- OrchestrationV2ThreadProjection,
11903
- "nodes" | "subagents" | "turnItems"
11904
- >;
11905
- readonly subagent: Pick<
11906
- Extract<OrchestrationV2TurnItem, { readonly type: "subagent" }>,
11907
- "subagentId" | "origin" | "childThreadId" | "providerInstanceId"
11908
- >;
11909
- }> = [{ projection: input.projection, subagent: item }];
11910
- for (const { projection, subagent } of dying) {
11911
- const ended = yield* endOrphanedNativeSubagent({
11912
- projection,
11913
- subagentId: subagent.subagentId,
11914
- status: "interrupted",
11915
- now: input.now,
11916
- emitted: yield* Ref.get(input.events),
11917
- allocateEventId,
11918
- }).pipe(mapDispatchError(input.command));
11919
- yield* Ref.update(input.events, (events) => [...events, ...ended]);
11920
- const childThreadId = subagent.childThreadId;
11921
- if (
11922
- subagent.origin !== "provider_native" ||
11923
- childThreadId === null ||
11924
- childThreadIds.has(childThreadId)
11925
- ) {
11926
- continue;
11927
- }
11928
- childThreadIds.add(childThreadId);
11929
- const child = yield* projectionStore
11930
- .getThreadRecords(childThreadId, ["nodes", "providerTurns", "subagents", "turnItems"], {
11931
- // Runless items are the subagent's; any run there is the user's own.
11932
- turnItemRunIds: [null],
11933
- turnItemStatuses: ["pending", "running", "waiting"],
11934
- })
11935
- .pipe(
11936
- Effect.catchTags({ ProjectionStoreThreadNotFoundError: () => Effect.succeed(null) }),
11937
- mapDispatchError(input.command),
11938
- );
11939
- if (child === null) continue;
11940
- const childEnded = yield* endRunlessRootTurns({
11941
- threadId: childThreadId,
11942
- providerInstanceId: subagent.providerInstanceId,
11943
- projection: child,
11944
- status: "interrupted",
11945
- now: input.now,
11946
- emitted: yield* Ref.get(input.events),
11947
- allocateEventId,
11948
- }).pipe(mapDispatchError(input.command));
11949
- yield* Ref.update(input.events, (events) => [...events, ...childEnded]);
11950
- for (const nested of child.turnItems) {
11951
- if (nested.type === "subagent") dying.push({ projection: child, subagent: nested });
11952
- }
11953
- for (const nested of child.subagents) {
11954
- if (nested.origin === "provider_native" && nested.runId === null) {
11955
- dying.push({
11956
- projection: child,
11957
- subagent: {
11958
- subagentId: nested.id,
11959
- origin: nested.origin,
11960
- childThreadId: nested.childThreadId,
11961
- providerInstanceId: nested.providerInstanceId,
11962
- },
11963
- });
11964
- }
11965
- }
11966
- }
11979
+ // A native subagent died with the process too.
11980
+ yield* endNativeSubagentTrees({
11981
+ command: input.command,
11982
+ events: input.events,
11983
+ dying: [{ projection: input.projection, subagent: item }],
11984
+ childThreadIds,
11985
+ now: input.now,
11986
+ });
11987
}
11988
for (const providerThread of input.projection.providerThreads) {
11989
if ((providerThread.pendingBackgroundTasks?.length ?? 0) === 0) continue;
@@ -13439,6 +13459,36 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
13459
for (const run of reached.values()) {
13460
yield* markStoppedRun({ command, events, thread: projection.thread, run, now });
13461
}
13462
+ // Record and item are written together, so an active record without a live item lost its end.
13463
+ const staleNative = (target === undefined ? projection.subagents : []).filter(
13464
+ (subagent) =>
13465
+ subagent.origin === "provider_native" &&
13466
+ subagent.runId !== null &&
13467
+ isOrchestrationV2WorkActive(subagent.status) &&
13468
+ !projection.turnItems.some(
13469
+ (item) => item.type === "subagent" && item.subagentId === subagent.id,
13470
+ ),
13471
+ );
13472
+ if (staleNative.length > 0) {
13473
+ const { nodes } = yield* projectionStore
13474
+ .getThreadRecords(command.threadId, ["nodes"])
13475
+ .pipe(mapDispatchError(command));
13476
+ const threadProjection = {
13477
+ nodes,
13478
+ subagents: projection.subagents,
13479
+ turnItems: projection.turnItems,
13480
+ };
13481
+ yield* endNativeSubagentTrees({
13482
+ command,
13483
+ events,
13484
+ dying: staleNative.map((subagent) => ({
13485
+ projection: threadProjection,
13486
+ subagent: nativeSubagentRef(subagent),
13487
+ })),
13488
+ childThreadIds: new Set(),
13489
+ now,
13490
+ });
13491
+ }
13492
yield* holdStoppedThread({ command, events, projection, cohortRunIds: [], now });
13493
yield* cancelIdleOrphans({ command, events, projection });
13494
yield* stopDelegatedTasks;
apps/server/src/orchestration-v2/ThreadStop.test.ts
+148
@@ -1839,6 +1839,154 @@ it.effect.each(["Stop", "Force stop"] as const)(
1839
}).pipe(Effect.provide(layerTest)),
1840
);
1841
1842
+// A finished delegated child whose native subagent the provider never reported ending,
1843
+// with no open turn item, and a nested native subagent on that agent's own thread.
1844
+it.effect(
1845
+ "Stop settles a delegated row whose finished child still records a running native subagent",
1846
+ () =>
1847
+ Effect.gen(function* () {
1848
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
1849
+ const projections = yield* ProjectionStore.ProjectionStoreV2;
1850
+ const parentThreadId = ThreadId.make("thread:held-row");
1851
+ yield* createWatchingThread(parentThreadId, 30);
1852
+ yield* send(parentThreadId, "parent", "start_immediately");
1853
+ const childThreadId = yield* delegate(parentThreadId, "held child");
1854
+ const child = yield* orchestrator.getThreadProjection(childThreadId);
1855
+ const childRun = child.runs[0]!;
1856
+ const task = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents[0]!;
1857
+ const now = yield* DateTime.now;
1858
+ const nativeThreadId = ThreadId.make("thread:held-row-native");
1859
+ const nestedThreadId = ThreadId.make("thread:held-row-nested");
1860
+ yield* createWatchingThread(nativeThreadId, 31);
1861
+ yield* createWatchingThread(nestedThreadId, 32);
1862
+ const nativeProviderThreadId = ProviderThreadId.make("provider-thread:held-row-native");
1863
+ const runningRoot = (threadId: ThreadId, id: NodeId) => ({
1864
+ ...child.nodes.find((node) => node.id === childRun.rootNodeId)!,
1865
+ id,
1866
+ rootNodeId: id,
1867
+ threadId,
1868
+ runId: null,
1869
+ parentNodeId: null,
1870
+ status: "running" as const,
1871
+ countsForRun: false,
1872
+ completedAt: null,
1873
+ });
1874
+ const nativeRoot = runningRoot(nativeThreadId, NodeId.make("node:held-row-native-root"));
1875
+ const nestedRoot = runningRoot(nestedThreadId, NodeId.make("node:held-row-nested-root"));
1876
+ for (const root of [nativeRoot, nestedRoot]) {
1877
+ yield* projections.apply({
1878
+ id: EventId.make(`event:held-row:${root.id}`),
1879
+ type: "node.updated",
1880
+ threadId: root.threadId,
1881
+ nodeId: root.id,
1882
+ occurredAt: now,
1883
+ payload: root,
1884
+ });
1885
+ }
1886
+ yield* projections.apply({
1887
+ id: EventId.make("event:held-row:native-turn"),
1888
+ type: "provider-turn.updated",
1889
+ threadId: nativeThreadId,
1890
+ nodeId: nativeRoot.id,
1891
+ occurredAt: now,
1892
+ payload: {
1893
+ id: ProviderTurnId.make("turn:held-row-native"),
1894
+ providerThreadId: nativeProviderThreadId,
1895
+ nodeId: nativeRoot.id,
1896
+ runAttemptId: null,
1897
+ nativeTurnRef: null,
1898
+ ordinal: 1,
1899
+ status: "running",
1900
+ startedAt: now,
1901
+ completedAt: null,
1902
+ },
1903
+ });
1904
+ const native = {
1905
+ ...task,
1906
+ id: NodeId.make("node:held-row-native"),
1907
+ threadId: childThreadId,
1908
+ runId: childRun.id,
1909
+ parentNodeId: childRun.rootNodeId!,
1910
+ origin: "provider_native" as const,
1911
+ providerThreadId: nativeProviderThreadId,
1912
+ childThreadId: nativeThreadId,
1913
+ status: "running" as const,
1914
+ completionDelivery: undefined,
1915
+ };
1916
+ yield* projections.apply({
1917
+ id: EventId.make("event:held-row:native"),
1918
+ type: "subagent.updated",
1919
+ threadId: childThreadId,
1920
+ runId: childRun.id,
1921
+ occurredAt: now,
1922
+ payload: native,
1923
+ });
1924
+ yield* projections.apply({
1925
+ id: EventId.make("event:held-row:nested"),
1926
+ type: "subagent.updated",
1927
+ threadId: nativeThreadId,
1928
+ occurredAt: now,
1929
+ payload: {
1930
+ ...native,
1931
+ id: NodeId.make("node:held-row-nested"),
1932
+ threadId: nativeThreadId,
1933
+ runId: null,
1934
+ parentNodeId: nativeRoot.id,
1935
+ childThreadId: nestedThreadId,
1936
+ },
1937
+ });
1938
+ yield* projections.apply({
1939
+ id: EventId.make("event:held-row:child-completed"),
1940
+ type: "run.updated",
1941
+ threadId: childThreadId,
1942
+ runId: childRun.id,
1943
+ occurredAt: now,
1944
+ payload: { ...childRun, status: "completed", startedAt: now, completedAt: now },
1945
+ });
1946
+ yield* orchestrator.recoverDelegatedTasks;
1947
+ assert.equal(
1948
+ (yield* orchestrator.getThreadProjection(parentThreadId)).subagents[0]?.status,
1949
+ "running",
1950
+ );
1951
+ const statuses = Effect.gen(function* () {
1952
+ const nativeAfter = yield* orchestrator.getThreadProjection(nativeThreadId);
1953
+ const nestedAfter = yield* orchestrator.getThreadProjection(nestedThreadId);
1954
+ return {
1955
+ row: (yield* orchestrator.getThreadProjection(parentThreadId)).subagents[0]?.status,
1956
+ native: (yield* orchestrator.getThreadProjection(childThreadId)).subagents.find(
1957
+ (subagent) => subagent.id === native.id,
1958
+ )?.status,
1959
+ nativeRoot: nativeAfter.nodes.find((node) => node.id === nativeRoot.id)?.status,
1960
+ nested: nativeAfter.subagents[0]?.status,
1961
+ nestedRoot: nestedAfter.nodes.find((node) => node.id === nestedRoot.id)?.status,
1962
+ };
1963
+ });
1964
+
1965
+ // The row's Stop: the parent's task, then the child thread.
1966
+ yield* orchestrator.dispatch({
1967
+ type: "subagent.stop",
1968
+ commandId: CommandId.make("held-row:stop-task"),
1969
+ threadId: parentThreadId,
1970
+ runId: task.runId!,
1971
+ subagentId: task.id,
1972
+ });
1973
+ yield* orchestrator.dispatch({
1974
+ type: "thread.stop",
1975
+ commandId: CommandId.make("held-row:stop-child"),
1976
+ threadId: childThreadId,
1977
+ });
1978
+ const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
1979
+ yield* worker.drain();
1980
+ assert.deepEqual(yield* statuses, {
1981
+ row: "completed",
1982
+ native: "interrupted",
1983
+ nativeRoot: "interrupted",
1984
+ nested: "interrupted",
1985
+ nestedRoot: "interrupted",
1986
+ });
1987
+ }).pipe(Effect.provide(layerTest)),
1988
+);
1989
+
1990
it.effect.each([
1991
{ followUp: "started its turn", startedBeforeRelease: true },
1992
{ followUp: "was promoted", startedBeforeRelease: false },