@devries-cameron20 / t3code-1 / commits / d32df2d34c
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): native session replacement fails honestly when the provider reuses its session (#27517)

Co-authored-by: TinBane <363069+TinBane@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: maria-rcks <maria@kuuro.net>

TinBane committed Nov 20, 2026 at 07:02 UTC d32df2d34c0927c12b05409ab05eae804366ae15
5 files changed +407 -3
apps/server/src/orchestration-v2/EffectWorker.test.ts
+1
@@ -124,6 +124,7 @@ function layerExecutorFor(input: {
124 detach: () => record("detach"),
125 otherWorkingThreads: () => Effect.succeed([]),
126 releaseUnlessBusy: () => Effect.succeed(true),
127 + forgetLoadedProviderThread: () => Effect.void,
128 subscribePromptSuggestion: () => Stream.empty,
129 }),
130 ),
apps/server/src/orchestration-v2/ProviderSessionManager.ts
+27
@@ -280,6 +280,17 @@ export interface ProviderSessionManagerV2Shape {
280 readonly providerSessionId: ProviderSessionId;
281 readonly threadId: ThreadId;
282 }) => Effect.Effect<ReadonlyArray<ThreadId>>;
283 + /**
284 + * Forgets that the thread has this provider thread loaded, so its next resume reaches the
285 + * adapter. Leaves the attachment, idle unload and MCP credential alone, unlike detach.
286 + */
287 + readonly forgetLoadedProviderThread: (input: {
288 + readonly providerSessionId: ProviderSessionId;
289 + readonly threadId: ThreadId;
290 + readonly providerThread: OrchestrationV2ProviderThread;
291 + readonly modelSelection: ModelSelection;
292 + readonly runtimePolicy: ProviderAdapter.ProviderAdapterV2RuntimePolicy;
293 + }) => Effect.Effect<void>;
294 /** The thread's latest predicted next prompt, then each replacement. */
295 readonly subscribePromptSuggestion: (
296 threadId: ThreadId,
@@ -3446,6 +3457,22 @@ export const layerWithOptions = (
3457 ),
3458 ),
3459 ),
3460 + forgetLoadedProviderThread: (input) =>
3461 + Ref.update(sessions, (current) => {
3462 + const key = sessionKey(input.providerSessionId);
3463 + const entry = current.get(key);
3464 + // Only the matching load: a different key belongs to a later load of this thread.
3465 + if (
3466 + entry === undefined ||
3467 + entry.loadedProviderThreadKeyByThread.get(input.threadId) !==
3468 + providerThreadLoadKey({ ...input, runtime: entry.runtime })
3469 + ) {
3470 + return current;
3471 + }
3472 + const loadedProviderThreadKeyByThread = new Map(entry.loadedProviderThreadKeyByThread);
3473 + loadedProviderThreadKeyByThread.delete(input.threadId);
3474 + return new Map(current).set(key, { ...entry, loadedProviderThreadKeyByThread });
3475 + }),
3476 // Stored suggestions can outlive a deleted thread, so check the thread before serving one.
3477 subscribePromptSuggestion: (threadId) =>
3478 SubscriptionRef.changes(promptSuggestionStore).pipe(
apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts
+1
@@ -218,6 +218,7 @@ function layerSessions(
218 detach: () => Effect.void,
219 otherWorkingThreads: () => Effect.succeed([]),
220 releaseUnlessBusy: () => Effect.succeed(true),
221 + forgetLoadedProviderThread: () => Effect.void,
222 subscribePromptSuggestion: () => Stream.empty,
223 }),
224 );
apps/server/src/orchestration-v2/ProviderTurnStartService.ts
+33 -3
@@ -1632,9 +1632,32 @@ export const layer: Layer.Layer<
1632 }),
1633 );
1634 if (replacement === undefined) return undefined;
1635 + if (
1636 + replacement.nativeThreadRef?.nativeId != null &&
1637 + replacement.nativeThreadRef.driver === providerThread.nativeThreadRef?.driver &&
1638 + replacement.nativeThreadRef.nativeId === providerThread.nativeThreadRef.nativeId
1639 + ) {
1640 + // Never retried. Forget the load first, or the next turn would skip resume and start on
1641 + // the reused session without this error or the recovery handoff.
1642 + yield* providerSessions.forgetLoadedProviderThread({
1643 + providerSessionId: session.providerSessionId,
1644 + threadId: projection.thread.id,
1645 + providerThread: replacement,
1646 + modelSelection: run.modelSelection,
1647 + runtimePolicy: resolvedRuntimePolicy,
1648 + });
1649 + yield* settleStartFailure({
1650 + signal: "provider-thread-load-failure",
1651 + title: "Provider turn failed to start",
1652 + error: new ProviderAdapter.ProviderAdapterProtocolError({
1653 + driver: session.driver,
1654 + detail:
1655 + "the provider returned its existing session when asked for a fresh one, so the conversation context could not be moved to a new session",
1656 + }),
1657 + });
1658 + return undefined;
1659 + }
1660 const { handoff, transferId, createdAt } = yield* recoveryHandoff;
1636 - effectiveHandoffs = withRecoveryHandoff(handoff);
1637 - resumeFellBack = true;
1661 const previousRun = projection.runs.findLast(
1662 (candidate) =>
1663 candidate.providerThreadId === providerThread.id &&
@@ -1679,7 +1702,11 @@ export const layer: Layer.Layer<
1702 strategy: handoff.strategy,
1703 summary: handoff.summaryText,
1704 };
1682 - yield* eventSink.write({
1705 + const replacementWrite = yield* eventSink.writeIfRunCurrent({
1706 + threadId: projection.thread.id,
1707 + runId: run.id,
1708 + activeAttemptId: attempt.id,
1709 + expectedStatus: "starting",
1710 events: [
1711 {
1712 id: yield* idAllocator.allocate.event({ threadId: projection.thread.id }),
@@ -1718,6 +1745,9 @@ export const layer: Layer.Layer<
1745 },
1746 ],
1747 });
1748 + if (!replacementWrite.committed) return undefined;
1749 + effectiveHandoffs = withRecoveryHandoff(handoff);
1750 + resumeFellBack = true;
1751 recoveryTurnItem = handoffTurnItem;
1752 return replacement;
1753 });
apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts
+345
@@ -138,6 +138,7 @@ function makeTestAdapter(input: {
138 readonly runtimeStarts?: Ref.Ref<ReadonlyArray<string>>;
139 // ACP cannot truncate, so its rollback answers with a fresh, empty native session.
140 readonly rollback?: "truncates" | "replaces_native_thread";
141 + readonly beforeForcedNativeThread?: Effect.Effect<void>;
142 }): ProviderAdapter.ProviderAdapterV2["Service"] {
143 return {
144 instanceId: input.instanceId,
@@ -210,6 +211,8 @@ function makeTestAdapter(input: {
211 if (threadInput.forceNewNativeThread && input.runtimeStarts !== undefined) {
212 yield* Ref.update(input.runtimeStarts, (starts) => [...starts, "replacement"]);
213 }
214 + if (threadInput.forceNewNativeThread && input.beforeForcedNativeThread !== undefined)
215 + yield* input.beforeForcedNativeThread;
216 const createdAt = yield* DateTime.now;
217 const generation =
218 input.nativeThreadGeneration === undefined
@@ -1772,6 +1775,348 @@ describe("orchestration v2 provider switching", () => {
1775 }),
1776 );
1777
1778 + const unverifiedReplacementHarness = <A, E, R>(
1779 + name: string,
1780 + adapters: ReadonlyArray<ProviderAdapter.ProviderAdapterV2["Service"]>,
1781 + body: (helpers: {
1782 + readonly orchestrator: Orchestrator.OrchestratorV2["Service"];
1783 + readonly worker: EffectWorker.OrchestrationEffectWorkerV2["Service"];
1784 + readonly dispatchRequest: (
1785 + ordinal: number,
1786 + selection: ModelSelection,
1787 + ) => ReturnType<Orchestrator.OrchestratorV2["Service"]["dispatch"]>;
1788 + readonly awaitSettled: (
1789 + ordinal: number,
1790 + ) => Effect.Effect<
1791 + number,
1792 + Orchestrator.OrchestratorV2Error | EffectWorker.OrchestrationEffectWorkerError
1793 + >;
1794 + }) => Effect.Effect<A, E, R>,
1795 + ) =>
1796 + Effect.scoped(
1797 + Effect.gen(function* () {
1798 + const cwd = yield* checkpointWorkspace(name);
1799 + return yield* Effect.gen(function* () {
1800 + const orchestrator = yield* Orchestrator.OrchestratorV2;
1801 + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
1802 + yield* orchestrator.dispatch({
1803 + type: "thread.create",
1804 + commandId: CommandId.make("unverified-replacement:create"),
1805 + threadId,
1806 + projectId,
1807 + createdBy: "user",
1808 + creationSource: "web",
1809 + title: "Unverified replacement",
1810 + modelSelection: CLAUDE_MODEL_SELECTION,
1811 + runtimeMode: "full-access",
1812 + interactionMode: "default",
1813 + branch: null,
1814 + worktreePath: null,
1815 + });
1816 + return yield* body({
1817 + orchestrator,
1818 + worker,
1819 + dispatchRequest: (ordinal, selection) =>
1820 + orchestrator.dispatch({
1821 + type: "message.dispatch",
1822 + commandId: CommandId.make(`unverified-replacement:${ordinal}`),
1823 + threadId,
1824 + messageId: MessageId.make(`unverified-replacement:${ordinal}`),
1825 + createdBy: "user",
1826 + creationSource: "web",
1827 + text: `Request ${ordinal}`,
1828 + attachments: [],
1829 + modelSelection: selection,
1830 + dispatchMode: { type: "start_immediately" },
1831 + }),
1832 + awaitSettled: (ordinal) =>
1833 + orchestrator.streamStoredEvents.pipe(
1834 + Stream.filter(
1835 + ({ event }) =>
1836 + event.type === "run.updated" &&
1837 + event.payload.ordinal === ordinal &&
1838 + (event.payload.status === "completed" ||
1839 + event.payload.status === "failed" ||
1840 + event.payload.status === "interrupted"),
1841 + ),
1842 + Stream.runHead,
1843 + Effect.andThen(worker.drain()),
1844 + ),
1845 + });
1846 + }).pipe(
1847 + Effect.provide(
1848 + ProviderReplayHarness.layerWithRegistry(
1849 + {
1850 + name,
1851 + runtimePolicyOverride: {
1852 + cwd,
1853 + approvalPolicy: "never",
1854 + sandboxPolicy: { type: "readOnly" },
1855 + },
1856 + },
1857 + ProviderAdapterRegistry.layerFromAdapters(adapters),
1858 + ),
1859 + ),
1860 + );
1861 + }),
1862 + );
1863 + // Replacement bookkeeping is a full-thread summary handoff plus a resume-fallback transfer.
1864 + const replacementRecordsFor = (projection: OrchestrationV2ThreadProjection, runId: string) => [
1865 + ...projection.contextHandoffs.filter(
1866 + (handoff) => handoff.targetRunId === runId && handoff.strategy === "full_thread_summary",
1867 + ),
1868 + ...projection.contextTransfers.filter(
1869 + (transfer) =>
1870 + transfer.targetRunId === runId && transfer.id.includes("provider_resume_fallback"),
1871 + ),
1872 + ];
1873 +
1874 + it.live("fails honestly when the provider reuses its session for a context handoff", () =>
1875 + Effect.gen(function* () {
1876 + const capturedTurns = yield* Ref.make<ReadonlyArray<CapturedTurn>>([]);
1877 + yield* unverifiedReplacementHarness(
1878 + "unverified-replacement-reused",
1879 + [
1880 + makeTestAdapter({
1881 + instanceId: CODEX_MODEL_SELECTION.instanceId,
1882 + driver: CODEX_DRIVER,
1883 + capabilities: CodexProviderCapabilitiesV2,
1884 + modelSelection: CODEX_MODEL_SELECTION,
1885 + responseByRunOrdinal: { 2: `Source work ${"s".repeat(12_000)}` },
1886 + capturedTurns,
1887 + }),
1888 + // Without a generation every ensureThread returns the same native id, like Muse.
1889 + makeTestAdapter({
1890 + instanceId: CLAUDE_MODEL_SELECTION.instanceId,
1891 + driver: CLAUDE_DRIVER,
1892 + capabilities: ClaudeProviderCapabilitiesV2,
1893 + modelSelection: CLAUDE_MODEL_SELECTION,
1894 + responseByRunOrdinal: {},
1895 + capturedTurns,
1896 + getModelContextWindow: () => 32_000,
1897 + }),
1898 + ],
1899 + ({ orchestrator, dispatchRequest, awaitSettled }) =>
1900 + Effect.gen(function* () {
1901 + yield* dispatchRequest(1, CLAUDE_MODEL_SELECTION);
1902 + yield* awaitSettled(1);
1903 + const target = (yield* orchestrator.getThreadProjection(threadId)).providerThreads[0]!;
1904 + // The native session has no room left for the returning handoff.
1905 + yield* (yield* EventSink.EventSinkV2).write({
1906 + events: [
1907 + {
1908 + id: EventId.make("unverified-replacement-usage"),
1909 + type: "provider-thread.updated",
1910 + threadId,
1911 + occurredAt: yield* DateTime.now,
1912 + payload: { ...target, contextUsage: { usedTokens: 31_000, maxTokens: 32_000 } },
1913 + },
1914 + ],
1915 + });
1916 + yield* dispatchRequest(2, CODEX_MODEL_SELECTION);
1917 + yield* awaitSettled(2);
1918 + yield* dispatchRequest(3, CLAUDE_MODEL_SELECTION);
1919 + yield* awaitSettled(3);
1920 +
1921 + const projection = yield* orchestrator.getThreadProjection(threadId);
1922 + const run3 = projection.runs.at(-1)!;
1923 + assert.equal(run3.status, "failed");
1924 + assert.deepEqual(replacementRecordsFor(projection, run3.id), []);
1925 + const failure = projection.turnItems.findLast((item) => item.type === "error");
1926 + assert.equal(
1927 + failure?.type === "error" ? failure.failure.message : null,
1928 + "claudeAgent provider protocol error: the provider returned its existing session when asked for a fresh one, so the conversation context could not be moved to a new session.",
1929 + );
1930 + assert.deepEqual(
1931 + (yield* Ref.get(capturedTurns)).map((turn) => turn.driver),
1932 + [CLAUDE_DRIVER, CODEX_DRIVER],
1933 + );
1934 + }),
1935 + );
1936 + }),
1937 + );
1938 +
1939 + it.live("fails without retrying when a failed resume's replacement reuses the session", () =>
1940 + Effect.gen(function* () {
1941 + const capturedTurns = yield* Ref.make<ReadonlyArray<CapturedTurn>>([]);
1942 + const failResumeOnce = yield* Ref.make(false);
1943 + yield* unverifiedReplacementHarness(
1944 + "unverified-replacement-reused-resume",
1945 + [
1946 + // Without a generation every ensureThread returns the same native id, like Muse.
1947 + makeTestAdapter({
1948 + instanceId: CLAUDE_MODEL_SELECTION.instanceId,
1949 + driver: CLAUDE_DRIVER,
1950 + capabilities: ClaudeProviderCapabilitiesV2,
1951 + modelSelection: CLAUDE_MODEL_SELECTION,
1952 + responseByRunOrdinal: {},
1953 + capturedTurns,
1954 + failResumeOnce,
1955 + }),
1956 + ],
1957 + ({ orchestrator, worker, dispatchRequest, awaitSettled }) =>
1958 + Effect.gen(function* () {
1959 + yield* dispatchRequest(1, CLAUDE_MODEL_SELECTION);
1960 + yield* awaitSettled(1);
1961 + const original = (yield* orchestrator.getThreadProjection(threadId))
1962 + .providerThreads[0]!;
1963 + yield* orchestrator.dispatch({
1964 + type: "provider-session.detach",
1965 + commandId: CommandId.make("unverified-replacement:detach"),
1966 + threadId,
1967 + providerSessionId: original.providerSessionId!,
1968 + });
1969 + yield* worker.drain();
1970 + // Only the first attempt fails to resume; a retry would find the session loaded.
1971 + yield* Ref.set(failResumeOnce, true);
1972 + yield* dispatchRequest(2, CLAUDE_MODEL_SELECTION);
1973 + yield* awaitSettled(2);
1974 +
1975 + const projection = yield* orchestrator.getThreadProjection(threadId);
1976 + const run2 = projection.runs.at(-1)!;
1977 + assert.equal(run2.status, "failed");
1978 + assert.deepEqual(replacementRecordsFor(projection, run2.id), []);
1979 + const failure = projection.turnItems.findLast((item) => item.type === "error");
1980 + assert.equal(
1981 + failure?.type === "error" ? failure.failure.message : null,
1982 + "claudeAgent provider protocol error: the provider returned its existing session when asked for a fresh one, so the conversation context could not be moved to a new session.",
1983 + );
1984 + assert.equal((yield* Ref.get(capturedTurns)).length, 1);
1985 + }),
1986 + );
1987 + }),
1988 + );
1989 +
1990 + it.live("keeps failing honestly on the next turn after rejecting a reused session", () =>
1991 + Effect.gen(function* () {
1992 + const capturedTurns = yield* Ref.make<ReadonlyArray<CapturedTurn>>([]);
1993 + yield* unverifiedReplacementHarness(
1994 + "unverified-replacement-reused-next-turn",
1995 + [
1996 + // Resume always fails and every ensureThread returns the same native id.
1997 + makeTestAdapter({
1998 + instanceId: CLAUDE_MODEL_SELECTION.instanceId,
1999 + driver: CLAUDE_DRIVER,
2000 + capabilities: ClaudeProviderCapabilitiesV2,
2001 + modelSelection: CLAUDE_MODEL_SELECTION,
2002 + responseByRunOrdinal: {},
2003 + capturedTurns,
2004 + failResume: true,
2005 + }),
2006 + ],
2007 + ({ orchestrator, worker, dispatchRequest, awaitSettled }) =>
2008 + Effect.gen(function* () {
2009 + yield* dispatchRequest(1, CLAUDE_MODEL_SELECTION);
2010 + yield* awaitSettled(1);
2011 + const original = (yield* orchestrator.getThreadProjection(threadId))
2012 + .providerThreads[0]!;
2013 + yield* orchestrator.dispatch({
2014 + type: "provider-session.detach",
2015 + commandId: CommandId.make("unverified-replacement:detach"),
2016 + threadId,
2017 + providerSessionId: original.providerSessionId!,
2018 + });
2019 + yield* worker.drain();
2020 + yield* dispatchRequest(2, CLAUDE_MODEL_SELECTION);
2021 + yield* awaitSettled(2);
2022 + // The rejected session must not count as loaded, or this turn would skip resume and
2023 + // run on it without the recovery history.
2024 + yield* dispatchRequest(3, CLAUDE_MODEL_SELECTION);
2025 + yield* awaitSettled(3);
2026 +
2027 + const projection = yield* orchestrator.getThreadProjection(threadId);
2028 + assert.deepEqual(
2029 + projection.runs.map((run) => run.status),
2030 + ["completed", "failed", "failed"],
2031 + );
2032 + assert.deepEqual(replacementRecordsFor(projection, projection.runs[2]!.id), []);
2033 + assert.equal((yield* Ref.get(capturedTurns)).length, 1);
2034 + }),
2035 + );
2036 + }),
2037 + );
2038 +
2039 + it.live("drops replacement bookkeeping when the run stops during replacement", () =>
2040 + Effect.gen(function* () {
2041 + const capturedTurns = yield* Ref.make<ReadonlyArray<CapturedTurn>>([]);
2042 + const failResumeOnce = yield* Ref.make(false);
2043 + const replacementRequested = yield* Deferred.make<void>();
2044 + const releaseReplacement = yield* Deferred.make<void>();
2045 + yield* unverifiedReplacementHarness(
2046 + "unverified-replacement-interrupted",
2047 + [
2048 + makeTestAdapter({
2049 + instanceId: CLAUDE_MODEL_SELECTION.instanceId,
2050 + driver: CLAUDE_DRIVER,
2051 + capabilities: ClaudeProviderCapabilitiesV2,
2052 + modelSelection: CLAUDE_MODEL_SELECTION,
2053 + responseByRunOrdinal: {},
2054 + capturedTurns,
2055 + failResumeOnce,
2056 + nativeThreadGeneration: yield* Ref.make(0),
2057 + beforeForcedNativeThread: Deferred.succeed(replacementRequested, undefined).pipe(
2058 + Effect.andThen(Deferred.await(releaseReplacement)),
2059 + ),
2060 + }),
2061 + ],
2062 + ({ orchestrator, worker, dispatchRequest, awaitSettled }) =>
2063 + Effect.gen(function* () {
2064 + yield* dispatchRequest(1, CLAUDE_MODEL_SELECTION);
2065 + yield* awaitSettled(1);
2066 + const original = (yield* orchestrator.getThreadProjection(threadId))
2067 + .providerThreads[0]!;
2068 + yield* orchestrator.dispatch({
2069 + type: "provider-session.detach",
2070 + commandId: CommandId.make("unverified-replacement:detach"),
2071 + threadId,
2072 + providerSessionId: original.providerSessionId!,
2073 + });
2074 + yield* worker.drain();
2075 + yield* Ref.set(failResumeOnce, true);
2076 + yield* dispatchRequest(2, CLAUDE_MODEL_SELECTION);
2077 + yield* Deferred.await(replacementRequested);
2078 + const run2 = (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!;
2079 + assert.equal(run2.status, "starting");
2080 + // run.interrupt would also cancel the start effect, so move the run on directly to
2081 + // reach the window where the replacement session exists but is not yet recorded.
2082 + const settledAt = yield* DateTime.now;
2083 + yield* (yield* EventSink.EventSinkV2).write({
2084 + events: [
2085 + {
2086 + id: EventId.make("unverified-replacement-interrupted"),
2087 + type: "run.updated",
2088 + threadId,
2089 + runId: run2.id,
2090 + occurredAt: settledAt,
2091 + payload: { ...run2, status: "interrupted", completedAt: settledAt },
2092 + },
2093 + ],
2094 + });
2095 + yield* Deferred.succeed(releaseReplacement, undefined);
2096 +
2097 + // Effects run one at a time per thread, so run 3 settles after run 2's start finishes.
2098 + // The next turn resumes the original native session with no abandoned replacement.
2099 + yield* dispatchRequest(3, CLAUDE_MODEL_SELECTION);
2100 + yield* awaitSettled(3);
2101 + const final = yield* orchestrator.getThreadProjection(threadId);
2102 + const run3 = final.runs.at(-1)!;
2103 + assert.equal(final.runs[1]?.status, "interrupted");
2104 + assert.deepEqual(replacementRecordsFor(final, run2.id), []);
2105 + assert.equal(run3.status, "completed");
2106 + assert.deepEqual(replacementRecordsFor(final, run3.id), []);
2107 + assert.equal(
2108 + final.providerThreads[0]?.nativeThreadRef?.nativeId,
2109 + original.nativeThreadRef?.nativeId,
2110 + );
2111 + const turns = yield* Ref.get(capturedTurns);
2112 + assert.equal(turns.length, 2);
2113 + assert.equal(turns[1]?.nativeThreadId, original.nativeThreadRef?.nativeId);
2114 + assert.include(turns[1]!.text, "Request 3");
2115 + }),
2116 + );
2117 + }),
2118 + );
2119 +
2120 it.live.each(["an older handoff", "a missed request"] as const)(
2121 "keeps the latest turn when a resume fallback also delivers %s",
2122 (older) =>