@devries-cameron20 / t3code Mirror · read-only / commits / 78b654f998
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 17:15 UTC

fix(server): Stop clears a finished delegated child's row held by a stale native subagent (#32307)

Julius Marminge committed Nov 13, 2026 at 10:37 UTC 78b654f998f651a9d7f00ecaf93499c5ed0bc5ad
2 files changed +273 -75
apps/server/src/orchestration-v2/Orchestrator.ts
+125 -75
@@ -1450,6 +1450,21 @@ function lastDeliveredRunForProviderThread(
1450 );
1451 }
1452
1453 +type DyingNativeSubagent = {
1454 + readonly projection: Pick<OrchestrationV2ThreadProjection, "nodes" | "subagents" | "turnItems">;
1455 + readonly subagent: Pick<
1456 + Extract<OrchestrationV2TurnItem, { readonly type: "subagent" }>,
1457 + "subagentId" | "origin" | "childThreadId" | "providerInstanceId"
1458 + >;
1459 +};
1460 +
1461 +const nativeSubagentRef = (subagent: OrchestrationV2Subagent): DyingNativeSubagent["subagent"] => ({
1462 + subagentId: subagent.id,
1463 + origin: subagent.origin,
1464 + childThreadId: subagent.childThreadId,
1465 + providerInstanceId: subagent.providerInstanceId,
1466 +});
1467 +
1468 const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(function* () {
1469 const checkpointService = yield* CheckpointServiceV2;
1470 const commandPolicy = yield* CommandPolicyV2;
@@ -11809,6 +11824,78 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
11824 }
11825 });
11826
11827 + // Ends each native subagent's record, its own thread's runless root turns, and the
11828 + // subagents it started there, which a provider can record on that thread rather than its parent's.
11829 + const endNativeSubagentTrees = Effect.fn("Orchestrator.endNativeSubagentTrees")(
11830 + function* (input: {
11831 + readonly command: Extract<
11832 + OrchestrationV2ServerCommand,
11833 + { readonly type: "run.interrupt" | "thread.background-work.settle" | "thread.stop" }
11834 + >;
11835 + readonly events: Ref.Ref<Array<OrchestrationV2DomainEvent>>;
11836 + readonly dying: ReadonlyArray<DyingNativeSubagent>;
11837 + readonly childThreadIds: Set<ThreadId>;
11838 + readonly now: DateTime.Utc;
11839 + }) {
11840 + const allocateEventId = () =>
11841 + idAllocator.allocate.event({
11842 + threadId: input.command.threadId,
11843 + commandId: input.command.commandId,
11844 + });
11845 + const dying = [...input.dying];
11846 + const { childThreadIds } = input;
11847 + for (const { projection, subagent } of dying) {
11848 + const ended = yield* endOrphanedNativeSubagent({
11849 + projection,
11850 + subagentId: subagent.subagentId,
11851 + status: "interrupted",
11852 + now: input.now,
11853 + emitted: yield* Ref.get(input.events),
11854 + allocateEventId,
11855 + }).pipe(mapDispatchError(input.command));
11856 + yield* Ref.update(input.events, (events) => [...events, ...ended]);
11857 + const childThreadId = subagent.childThreadId;
11858 + if (
11859 + subagent.origin !== "provider_native" ||
11860 + childThreadId === null ||
11861 + childThreadIds.has(childThreadId)
11862 + ) {
11863 + continue;
11864 + }
11865 + childThreadIds.add(childThreadId);
11866 + const child = yield* projectionStore
11867 + .getThreadRecords(childThreadId, ["nodes", "providerTurns", "subagents", "turnItems"], {
11868 + // Runless items are the subagent's; any run there is the user's own.
11869 + turnItemRunIds: [null],
11870 + turnItemStatuses: ["pending", "running", "waiting"],
11871 + })
11872 + .pipe(
11873 + Effect.catchTags({ ProjectionStoreThreadNotFoundError: () => Effect.succeed(null) }),
11874 + mapDispatchError(input.command),
11875 + );
11876 + if (child === null) continue;
11877 + const childEnded = yield* endRunlessRootTurns({
11878 + threadId: childThreadId,
11879 + providerInstanceId: subagent.providerInstanceId,
11880 + projection: child,
11881 + status: "interrupted",
11882 + now: input.now,
11883 + emitted: yield* Ref.get(input.events),
11884 + allocateEventId,
11885 + }).pipe(mapDispatchError(input.command));
11886 + yield* Ref.update(input.events, (events) => [...events, ...childEnded]);
11887 + for (const nested of child.turnItems) {
11888 + if (nested.type === "subagent") dying.push({ projection: child, subagent: nested });
11889 + }
11890 + for (const nested of child.subagents) {
11891 + if (nested.origin === "provider_native" && nested.runId === null) {
11892 + dying.push({ projection: child, subagent: nativeSubagentRef(nested) });
11893 + }
11894 + }
11895 + }
11896 + },
11897 + );
11898 +
11899 const settleBackgroundWork = Effect.fn("Orchestrator.settleBackgroundWork")(function* (input: {
11900 readonly command: Extract<
11901 OrchestrationV2ServerCommand,
@@ -11825,11 +11912,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
11912 }) {
11913 const emitEvent = emit(input.events, input.command);
11914 const runOrdinals = new Map(input.projection.runs.map((run) => [run.id, run.ordinal]));
11828 - const allocateEventId = () =>
11829 - idAllocator.allocate.event({
11830 - threadId: input.command.threadId,
11831 - commandId: input.command.commandId,
11832 - });
11915 const childThreadIds = new Set<ThreadId>();
11916 const liveness = new Map<string, boolean>();
11917 const hasLiveSession = Effect.fn("Orchestrator.hasLiveSession")(function* (
@@ -11894,76 +11976,14 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
11976 },
11977 });
11978 if (item.type !== "subagent") continue;
11897 - // A native subagent died with the process too: end its record, its own
11898 - // thread's runless root turn, and the subagents it started there, which
11899 - // a provider can record on that thread rather than this one.
11900 - const dying: Array<{
11901 - readonly projection: Pick<
11902 - OrchestrationV2ThreadProjection,
11903 - "nodes" | "subagents" | "turnItems"
11904 - >;
11905 - readonly subagent: Pick<
11906 - Extract<OrchestrationV2TurnItem, { readonly type: "subagent" }>,
11907 - "subagentId" | "origin" | "childThreadId" | "providerInstanceId"
11908 - >;
11909 - }> = [{ projection: input.projection, subagent: item }];
11910 - for (const { projection, subagent } of dying) {
11911 - const ended = yield* endOrphanedNativeSubagent({
11912 - projection,
11913 - subagentId: subagent.subagentId,
11914 - status: "interrupted",
11915 - now: input.now,
11916 - emitted: yield* Ref.get(input.events),
11917 - allocateEventId,
11918 - }).pipe(mapDispatchError(input.command));
11919 - yield* Ref.update(input.events, (events) => [...events, ...ended]);
11920 - const childThreadId = subagent.childThreadId;
11921 - if (
11922 - subagent.origin !== "provider_native" ||
11923 - childThreadId === null ||
11924 - childThreadIds.has(childThreadId)
11925 - ) {
11926 - continue;
11927 - }
11928 - childThreadIds.add(childThreadId);
11929 - const child = yield* projectionStore
11930 - .getThreadRecords(childThreadId, ["nodes", "providerTurns", "subagents", "turnItems"], {
11931 - // Runless items are the subagent's; any run there is the user's own.
11932 - turnItemRunIds: [null],
11933 - turnItemStatuses: ["pending", "running", "waiting"],
11934 - })
11935 - .pipe(
11936 - Effect.catchTags({ ProjectionStoreThreadNotFoundError: () => Effect.succeed(null) }),
11937 - mapDispatchError(input.command),
11938 - );
11939 - if (child === null) continue;
11940 - const childEnded = yield* endRunlessRootTurns({
11941 - threadId: childThreadId,
11942 - providerInstanceId: subagent.providerInstanceId,
11943 - projection: child,
11944 - status: "interrupted",
11945 - now: input.now,
11946 - emitted: yield* Ref.get(input.events),
11947 - allocateEventId,
11948 - }).pipe(mapDispatchError(input.command));
11949 - yield* Ref.update(input.events, (events) => [...events, ...childEnded]);
11950 - for (const nested of child.turnItems) {
11951 - if (nested.type === "subagent") dying.push({ projection: child, subagent: nested });
11952 - }
11953 - for (const nested of child.subagents) {
11954 - if (nested.origin === "provider_native" && nested.runId === null) {
11955 - dying.push({
11956 - projection: child,
11957 - subagent: {
11958 - subagentId: nested.id,
11959 - origin: nested.origin,
11960 - childThreadId: nested.childThreadId,
11961 - providerInstanceId: nested.providerInstanceId,
11962 - },
11963 - });
11964 - }
11965 - }
11966 - }
11979 + // A native subagent died with the process too.
11980 + yield* endNativeSubagentTrees({
11981 + command: input.command,
11982 + events: input.events,
11983 + dying: [{ projection: input.projection, subagent: item }],
11984 + childThreadIds,
11985 + now: input.now,
11986 + });
11987 }
11988 for (const providerThread of input.projection.providerThreads) {
11989 if ((providerThread.pendingBackgroundTasks?.length ?? 0) === 0) continue;
@@ -13439,6 +13459,36 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
13459 for (const run of reached.values()) {
13460 yield* markStoppedRun({ command, events, thread: projection.thread, run, now });
13461 }
13462 + // Record and item are written together, so an active record without a live item lost its end.
13463 + const staleNative = (target === undefined ? projection.subagents : []).filter(
13464 + (subagent) =>
13465 + subagent.origin === "provider_native" &&
13466 + subagent.runId !== null &&
13467 + isOrchestrationV2WorkActive(subagent.status) &&
13468 + !projection.turnItems.some(
13469 + (item) => item.type === "subagent" && item.subagentId === subagent.id,
13470 + ),
13471 + );
13472 + if (staleNative.length > 0) {
13473 + const { nodes } = yield* projectionStore
13474 + .getThreadRecords(command.threadId, ["nodes"])
13475 + .pipe(mapDispatchError(command));
13476 + const threadProjection = {
13477 + nodes,
13478 + subagents: projection.subagents,
13479 + turnItems: projection.turnItems,
13480 + };
13481 + yield* endNativeSubagentTrees({
13482 + command,
13483 + events,
13484 + dying: staleNative.map((subagent) => ({
13485 + projection: threadProjection,
13486 + subagent: nativeSubagentRef(subagent),
13487 + })),
13488 + childThreadIds: new Set(),
13489 + now,
13490 + });
13491 + }
13492 yield* holdStoppedThread({ command, events, projection, cohortRunIds: [], now });
13493 yield* cancelIdleOrphans({ command, events, projection });
13494 yield* stopDelegatedTasks;
apps/server/src/orchestration-v2/ThreadStop.test.ts
+148
@@ -1839,6 +1839,154 @@ it.effect.each(["Stop", "Force stop"] as const)(
1839 }).pipe(Effect.provide(layerTest)),
1840 );
1841
1842 +// A finished delegated child whose native subagent the provider never reported ending,
1843 +// with no open turn item, and a nested native subagent on that agent's own thread.
1844 +it.effect(
1845 + "Stop settles a delegated row whose finished child still records a running native subagent",
1846 + () =>
1847 + Effect.gen(function* () {
1848 + const orchestrator = yield* Orchestrator.OrchestratorV2;
1849 + const projections = yield* ProjectionStore.ProjectionStoreV2;
1850 + const parentThreadId = ThreadId.make("thread:held-row");
1851 + yield* createWatchingThread(parentThreadId, 30);
1852 + yield* send(parentThreadId, "parent", "start_immediately");
1853 + const childThreadId = yield* delegate(parentThreadId, "held child");
1854 + const child = yield* orchestrator.getThreadProjection(childThreadId);
1855 + const childRun = child.runs[0]!;
1856 + const task = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents[0]!;
1857 + const now = yield* DateTime.now;
1858 + const nativeThreadId = ThreadId.make("thread:held-row-native");
1859 + const nestedThreadId = ThreadId.make("thread:held-row-nested");
1860 + yield* createWatchingThread(nativeThreadId, 31);
1861 + yield* createWatchingThread(nestedThreadId, 32);
1862 + const nativeProviderThreadId = ProviderThreadId.make("provider-thread:held-row-native");
1863 + const runningRoot = (threadId: ThreadId, id: NodeId) => ({
1864 + ...child.nodes.find((node) => node.id === childRun.rootNodeId)!,
1865 + id,
1866 + rootNodeId: id,
1867 + threadId,
1868 + runId: null,
1869 + parentNodeId: null,
1870 + status: "running" as const,
1871 + countsForRun: false,
1872 + completedAt: null,
1873 + });
1874 + const nativeRoot = runningRoot(nativeThreadId, NodeId.make("node:held-row-native-root"));
1875 + const nestedRoot = runningRoot(nestedThreadId, NodeId.make("node:held-row-nested-root"));
1876 + for (const root of [nativeRoot, nestedRoot]) {
1877 + yield* projections.apply({
1878 + id: EventId.make(`event:held-row:${root.id}`),
1879 + type: "node.updated",
1880 + threadId: root.threadId,
1881 + nodeId: root.id,
1882 + occurredAt: now,
1883 + payload: root,
1884 + });
1885 + }
1886 + yield* projections.apply({
1887 + id: EventId.make("event:held-row:native-turn"),
1888 + type: "provider-turn.updated",
1889 + threadId: nativeThreadId,
1890 + nodeId: nativeRoot.id,
1891 + occurredAt: now,
1892 + payload: {
1893 + id: ProviderTurnId.make("turn:held-row-native"),
1894 + providerThreadId: nativeProviderThreadId,
1895 + nodeId: nativeRoot.id,
1896 + runAttemptId: null,
1897 + nativeTurnRef: null,
1898 + ordinal: 1,
1899 + status: "running",
1900 + startedAt: now,
1901 + completedAt: null,
1902 + },
1903 + });
1904 + const native = {
1905 + ...task,
1906 + id: NodeId.make("node:held-row-native"),
1907 + threadId: childThreadId,
1908 + runId: childRun.id,
1909 + parentNodeId: childRun.rootNodeId!,
1910 + origin: "provider_native" as const,
1911 + providerThreadId: nativeProviderThreadId,
1912 + childThreadId: nativeThreadId,
1913 + status: "running" as const,
1914 + completionDelivery: undefined,
1915 + };
1916 + yield* projections.apply({
1917 + id: EventId.make("event:held-row:native"),
1918 + type: "subagent.updated",
1919 + threadId: childThreadId,
1920 + runId: childRun.id,
1921 + occurredAt: now,
1922 + payload: native,
1923 + });
1924 + yield* projections.apply({
1925 + id: EventId.make("event:held-row:nested"),
1926 + type: "subagent.updated",
1927 + threadId: nativeThreadId,
1928 + occurredAt: now,
1929 + payload: {
1930 + ...native,
1931 + id: NodeId.make("node:held-row-nested"),
1932 + threadId: nativeThreadId,
1933 + runId: null,
1934 + parentNodeId: nativeRoot.id,
1935 + childThreadId: nestedThreadId,
1936 + },
1937 + });
1938 + yield* projections.apply({
1939 + id: EventId.make("event:held-row:child-completed"),
1940 + type: "run.updated",
1941 + threadId: childThreadId,
1942 + runId: childRun.id,
1943 + occurredAt: now,
1944 + payload: { ...childRun, status: "completed", startedAt: now, completedAt: now },
1945 + });
1946 + yield* orchestrator.recoverDelegatedTasks;
1947 + assert.equal(
1948 + (yield* orchestrator.getThreadProjection(parentThreadId)).subagents[0]?.status,
1949 + "running",
1950 + );
1951 + const statuses = Effect.gen(function* () {
1952 + const nativeAfter = yield* orchestrator.getThreadProjection(nativeThreadId);
1953 + const nestedAfter = yield* orchestrator.getThreadProjection(nestedThreadId);
1954 + return {
1955 + row: (yield* orchestrator.getThreadProjection(parentThreadId)).subagents[0]?.status,
1956 + native: (yield* orchestrator.getThreadProjection(childThreadId)).subagents.find(
1957 + (subagent) => subagent.id === native.id,
1958 + )?.status,
1959 + nativeRoot: nativeAfter.nodes.find((node) => node.id === nativeRoot.id)?.status,
1960 + nested: nativeAfter.subagents[0]?.status,
1961 + nestedRoot: nestedAfter.nodes.find((node) => node.id === nestedRoot.id)?.status,
1962 + };
1963 + });
1964 +
1965 + // The row's Stop: the parent's task, then the child thread.
1966 + yield* orchestrator.dispatch({
1967 + type: "subagent.stop",
1968 + commandId: CommandId.make("held-row:stop-task"),
1969 + threadId: parentThreadId,
1970 + runId: task.runId!,
1971 + subagentId: task.id,
1972 + });
1973 + yield* orchestrator.dispatch({
1974 + type: "thread.stop",
1975 + commandId: CommandId.make("held-row:stop-child"),
1976 + threadId: childThreadId,
1977 + });
1978 + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
1979 + yield* worker.drain();
1980 + assert.deepEqual(yield* statuses, {
1981 + row: "completed",
1982 + native: "interrupted",
1983 + nativeRoot: "interrupted",
1984 + nested: "interrupted",
1985 + nestedRoot: "interrupted",
1986 + });
1987 + }).pipe(Effect.provide(layerTest)),
1988 +);
1989 +
1990 it.effect.each([
1991 { followUp: "started its turn", startedBeforeRelease: true },
1992 { followUp: "was promoted", startedBeforeRelease: false },