@devries-cameron20 / t3code Mirror · read-only / commits / 4bf8b476fa
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 23:15 UTC

fix(provider-muse): relaunch a host that died between turns (#19195)

Co-authored-by: dan <dan@bw-hq-linuxws01.bellwetherit.com> Co-authored-by: Julius Marminge <julius0216@outlook.com>

Dan Donathan committed Oct 19, 2026 at 22:03 UTC 4bf8b476fa749708b7c1343ba9c9959edb2de8b9
4 files changed +318 -8
apps/server/src/orchestration-v2/ProviderSessionManager.test.ts
+121
@@ -4446,6 +4446,127 @@ it.effect(
4446 }),
4447 );
4448
4449 +it.effect(
4450 + "ProviderSessionManagerV2 invalidates only errored threads and keeps reusable session errors cached",
4451 + () =>
4452 + Effect.gen(function* () {
4453 + const state = yield* Ref.make(emptyState);
4454 + const effect = Effect.gen(function* () {
4455 + const eventSink = yield* EventSink.EventSinkV2;
4456 + const idAllocator = yield* IdAllocator.IdAllocatorV2;
4457 + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
4458 + const now = yield* DateTime.now;
4459 + const openCachedRuntime = Effect.fnUntraced(function* (
4460 + threadId: ThreadId,
4461 + sharedSessionId?: ProviderSessionId,
4462 + ) {
4463 + const providerSessionId =
4464 + sharedSessionId ??
4465 + (yield* idAllocator.allocate.providerSession({
4466 + providerInstanceId: modelSelection.instanceId,
4467 + threadId,
4468 + }));
4469 + yield* eventSink.write({
4470 + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })],
4471 + });
4472 + const runtime = yield* manager.open({
4473 + threadId,
4474 + providerSessionId,
4475 + modelSelection,
4476 + runtimePolicy,
4477 + });
4478 + const providerThread = makeProviderThread({
4479 + idAllocator,
4480 + threadId,
4481 + providerSessionId,
4482 + now,
4483 + nativeThreadId: String(threadId),
4484 + });
4485 + const resume = runtime.resumeThread({
4486 + providerThread,
4487 + threadId,
4488 + modelSelection,
4489 + runtimePolicy,
4490 + });
4491 + yield* resume;
4492 + yield* resume;
4493 + return { runtime, providerThread, resume };
4494 + });
4495 + const failed = yield* openCachedRuntime(ThreadId.make("idle-failed-runtime"));
4496 + const healthy = yield* openCachedRuntime(ThreadId.make("idle-healthy-runtime"));
4497 + const sibling = yield* openCachedRuntime(
4498 + ThreadId.make("idle-healthy-sibling"),
4499 + failed.runtime.providerSessionId,
4500 + );
4501 + assert.strictEqual(sibling.runtime, failed.runtime);
4502 + assert.equal((yield* Ref.get(state)).openCount, 2);
4503 + assert.equal((yield* Ref.get(state)).resumeCount, 3);
4504 +
4505 + const queue = (yield* Ref.get(state)).eventQueues.get(
4506 + String(failed.runtime.providerSessionId),
4507 + );
4508 + assert.isDefined(queue);
4509 + const publish = Effect.fnUntraced(function* (
4510 + event: ProviderAdapter.ProviderAdapterV2Event,
4511 + ) {
4512 + const subscription = yield* failed.runtime.subscribeEvents!;
4513 + yield* Queue.offer(queue!, event);
4514 + // Observing the exact event ensures the manager has handled it before resuming.
4515 + assert.deepEqual(yield* subscription.events.pipe(Stream.runHead), Option.some(event));
4516 + });
4517 + const countRawResumes = Effect.fnUntraced(function* (cached: typeof failed) {
4518 + const before = (yield* Ref.get(state)).resumeCount;
4519 + yield* cached.resume;
4520 + return (yield* Ref.get(state)).resumeCount - before;
4521 + });
4522 + yield* publish({
4523 + type: "provider_thread.updated",
4524 + driver: CODEX_DRIVER,
4525 + providerThread: failed.providerThread,
4526 + });
4527 + yield* publish({
4528 + type: "provider_session.updated",
4529 + driver: CODEX_DRIVER,
4530 + providerSession: {
4531 + ...failed.runtime.providerSession,
4532 + status: "error",
4533 + lastError: "Reusable turn failure",
4534 + },
4535 + });
4536 + const afterSessionError = {
4537 + reusableThread: yield* countRawResumes(failed),
4538 + siblingThread: yield* countRawResumes(sibling),
4539 + };
4540 +
4541 + yield* publish({
4542 + type: "provider_thread.updated",
4543 + driver: CODEX_DRIVER,
4544 + providerThread: { ...failed.providerThread, status: "error" },
4545 + });
4546 + const afterThreadError = {
4547 + healthyRuntime: yield* countRawResumes(healthy),
4548 + siblingThread: yield* countRawResumes(sibling),
4549 + failedThread: yield* countRawResumes(failed),
4550 + failedThreadAgain: yield* countRawResumes(failed),
4551 + };
4552 + assert.deepEqual(
4553 + { afterSessionError, afterThreadError },
4554 + {
4555 + afterSessionError: { reusableThread: 0, siblingThread: 0 },
4556 + afterThreadError: {
4557 + healthyRuntime: 0,
4558 + siblingThread: 0,
4559 + failedThread: 1,
4560 + failedThreadAgain: 0,
4561 + },
4562 + },
4563 + );
4564 + });
4565 +
4566 + yield* effect.pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 1000 })));
4567 + }),
4568 +);
4569 +
4570 it.effect(
4571 "ProviderSessionManagerV2 re-attaching a thread waits for its in-flight unload, then reloads it",
4572 () =>
apps/server/src/orchestration-v2/ProviderSessionManager.ts
+19
@@ -1969,6 +1969,25 @@ export const layerWithOptions = (
1969 ),
1970 Effect.andThen(
1971 Effect.gen(function* () {
1972 + if (
1973 + event.type === "provider_thread.updated" &&
1974 + event.providerThread.status === "error"
1975 + ) {
1976 + const threadId = event.providerThread.appThreadId;
1977 + yield* Ref.update(sessions, (sessions) => {
1978 + const key = sessionKey(entry.runtime.providerSessionId);
1979 + const latest = sessions.get(key);
1980 + if (latest?.runtime !== entry.runtime || threadId === null) return sessions;
1981 + const loadedProviderThreadKeyByThread = new Map(
1982 + latest.loadedProviderThreadKeyByThread,
1983 + );
1984 + loadedProviderThreadKeyByThread.delete(threadId);
1985 + return new Map(sessions).set(key, {
1986 + ...latest,
1987 + loadedProviderThreadKeyByThread,
1988 + });
1989 + });
1990 + }
1991 // Some providers can block before a run subscriber exists
1992 // (project trust, login, or session-switch hooks). Persist
1993 // their runless request artifacts directly so the normal T3
packages/provider-muse/src/server/adapter.test.ts
+158 -4
@@ -832,7 +832,25 @@ describe("MuseAdapterV2", () => {
832 Effect.gen(function* () {
833 const fake = yield* makeFakeMuse();
834 const harness = yield* makeHarness(fake);
835 - const { nativeId } = yield* startConversation(harness, fake);
835 + const first = yield* startConversation(harness, fake);
836 + yield* fake.emit("item/started", {
837 + turnId: first.nativeId,
838 + item: {
839 + itemId: "background-child",
840 + kind: "subagent",
841 + revision: 1,
842 + status: "inProgress",
843 + turnId: first.nativeId,
844 + subagentId: "subagent-1",
845 + controlStatus: "running",
846 + childSessionId: "child-session-1",
847 + objective: "Finish in the background",
848 + },
849 + });
850 + const child = yield* harness.takeEvent("app_thread.created");
851 + yield* fake.emit("turn/completed", { turnId: first.nativeId, terminal: "completed" });
852 + yield* harness.takeEvent("turn.terminal");
853 + const { nativeId } = yield* startConversation(harness, fake, 2);
854 yield* fake.emit("approval/requested", approval(nativeId));
855 yield* harness.takeEvent(
856 "runtime_request.updated",
@@ -848,15 +866,36 @@ describe("MuseAdapterV2", () => {
866 event.type === "runtime_request.updated" && event.runtimeRequest.status === "cancelled",
867 ),
868 );
869 + const streamExit = yield* Deferred.await(harness.eventsEnded);
870 + assert.strictEqual(streamExit._tag, "Failure");
871 + assert.isTrue(
872 + harness.allEvents.some(
873 + (event) =>
874 + event.type === "subagent.updated" &&
875 + event.subagent.childThreadId === child.appThread.id &&
876 + event.subagent.status === "failed",
877 + ),
878 + );
879 + assert.isTrue(
880 + harness.allEvents.some(
881 + (event) =>
882 + event.type === "node.updated" &&
883 + event.node.threadId === child.appThread.id &&
884 + event.node.kind === "root_turn" &&
885 + event.node.status === "failed",
886 + ),
887 + );
888 const result = yield* harness.runtime
852 - .startTurn(yield* turnInput(harness.providerThread, 2))
889 + .startTurn(yield* turnInput(harness.providerThread, 3))
890 .pipe(Effect.exit);
891 assert.strictEqual(result._tag, "Failure");
892 assert.strictEqual(
856 - harness.allEvents.filter((event) => event.type === "turn.terminal").length,
893 + harness.allEvents.filter(
894 + (event) => event.type === "turn.terminal" && event.runOrdinal === 2,
895 + ).length,
896 1,
897 );
859 - assert.strictEqual(fake.calls.filter((call) => call.method === "turn/start").length, 1);
898 + assert.strictEqual(fake.calls.filter((call) => call.method === "turn/start").length, 2);
899 }).pipe(Effect.scoped, Effect.provide(testLayer)),
900 );
901
@@ -1167,6 +1206,92 @@ describe("MuseAdapterV2", () => {
1206 }).pipe(Effect.scoped, Effect.provide(testLayer)),
1207 );
1208
1209 + it.effect.each([
1210 + { resume: true, label: "resumes the saved session" },
1211 + { resume: false, label: "starts a fresh session" },
1212 + ])("completes a turn after an idle host dies and $label", ({ resume }) =>
1213 + Effect.gen(function* () {
1214 + const fake = yield* makeFakeMuse("first");
1215 + const recovery = yield* makeFakeMuse("second");
1216 + const harness = yield* makeHarness(fake, INSTANCE_ID, "saved-session", recovery);
1217 + yield* fake.disconnect();
1218 + yield* harness.takeEvent(
1219 + "provider_session.updated",
1220 + (event) => event.providerSession.status === "error",
1221 + );
1222 +
1223 + const reEnsured = yield* harness.runtime.ensureThread({
1224 + threadId: THREAD_ID,
1225 + modelSelection: modelSelection(),
1226 + runtimePolicy,
1227 + existingProviderThread: resume
1228 + ? harness.providerThread
1229 + : { ...harness.providerThread, nativeThreadRef: null },
1230 + });
1231 + const registered = yield* recovery.takeCall(resume ? "session/resume" : "session/start");
1232 + assert.strictEqual(registered.params.sessionId, reEnsured.nativeThreadRef?.nativeId);
1233 + assert.strictEqual(reEnsured.id, harness.providerThread.id);
1234 + assert.strictEqual(reEnsured.nativeThreadRef?.nativeId === "saved-session", resume);
1235 +
1236 + yield* harness.runtime.startTurn(yield* turnInput(reEnsured));
1237 + const started = yield* recovery.takeCall("turn/start");
1238 + yield* recovery.emit("turn/completed", {
1239 + turnId: started.commandId,
1240 + terminal: "completed",
1241 + });
1242 + const terminal = yield* harness.takeEvent("turn.terminal");
1243 + assert.strictEqual(terminal.status, "completed");
1244 + assert.strictEqual(terminal.providerThreadId, reEnsured.id);
1245 + assert.strictEqual(terminal.threadDisposition, "reusable");
1246 + }).pipe(Effect.scoped, Effect.provide(testLayer)),
1247 + );
1248 +
1249 + it.effect("discards a dead host's pending continuation before relaunching", () =>
1250 + Effect.gen(function* () {
1251 + const fake = yield* makeFakeMuse("first");
1252 + const recovery = yield* makeFakeMuse("second");
1253 + const offered =
1254 + yield* Deferred.make<ProviderContinuationRequests.ProviderContinuationRequest>();
1255 + const harness = yield* makeHarness(
1256 + fake,
1257 + INSTANCE_ID,
1258 + "saved-session",
1259 + recovery,
1260 + undefined,
1261 + runtimePolicy,
1262 + { continuationRequests: { offer: (request) => Deferred.succeed(offered, request) } },
1263 + );
1264 + yield* fake.emit("turn/started", { turnId: "dead-host-report" });
1265 + const continuation = yield* Deferred.await(offered);
1266 + assert.isTrue(yield* harness.runtime.hasPendingBackgroundWork!);
1267 + yield* fake.disconnect();
1268 + yield* harness.takeEvent(
1269 + "provider_session.updated",
1270 + (event) => event.providerSession.status === "error",
1271 + );
1272 + assert.isFalse(yield* harness.runtime.hasPendingBackgroundWork!);
1273 + const dispatched = yield* continuation.dispatchIfCurrent!(
1274 + Effect.die("Stale continuation dispatched"),
1275 + );
1276 + assert.isTrue(Option.isNone(dispatched));
1277 +
1278 + const reEnsured = yield* harness.runtime.ensureThread({
1279 + threadId: THREAD_ID,
1280 + modelSelection: modelSelection(),
1281 + runtimePolicy,
1282 + existingProviderThread: harness.providerThread,
1283 + });
1284 + const input = yield* turnInput(reEnsured);
1285 + yield* harness.runtime.startTurn({
1286 + ...input,
1287 + message: { ...input.message, createdBy: "agent", creationSource: "provider" },
1288 + });
1289 + const running = yield* harness.takeEvent("provider_turn.updated");
1290 + assert.notStrictEqual(running.providerTurn.nativeTurnRef?.nativeId, "dead-host-report");
1291 + assert.strictEqual((yield* harness.takeEvent("turn.terminal")).status, "completed");
1292 + }).pipe(Effect.scoped, Effect.provide(testLayer)),
1293 + );
1294 +
1295 it.effect("serializes interruption behind pending turn admission without losing the target", () =>
1296 Effect.gen(function* () {
1297 const fake = yield* makeFakeMuse();
@@ -1196,6 +1321,35 @@ describe("MuseAdapterV2", () => {
1321 }).pipe(Effect.scoped, Effect.provide(testLayer)),
1322 );
1323
1324 + it.effect("keeps an intentionally interrupted compaction closed on ensureThread", () =>
1325 + Effect.gen(function* () {
1326 + const fake = yield* makeFakeMuse();
1327 + const recovery = yield* makeFakeMuse("replacement");
1328 + const harness = yield* makeHarness(fake, INSTANCE_ID, undefined, recovery);
1329 + yield* harness.runtime.compactThread!(yield* turnInput(harness.providerThread));
1330 + const running = yield* harness.takeEvent("provider_turn.updated");
1331 + yield* harness.runtime.interruptTurn({
1332 + providerThread: harness.providerThread,
1333 + providerTurnId: running.providerTurn.id,
1334 + });
1335 + const terminal = yield* harness.takeEvent("turn.terminal");
1336 + assert.strictEqual(terminal.status, "interrupted");
1337 + assert.strictEqual(terminal.threadDisposition, "broken");
1338 + assert.strictEqual((yield* Deferred.await(harness.eventsEnded))._tag, "Success");
1339 +
1340 + const result = yield* harness.runtime
1341 + .ensureThread({
1342 + threadId: THREAD_ID,
1343 + modelSelection: modelSelection(),
1344 + runtimePolicy,
1345 + existingProviderThread: harness.providerThread,
1346 + })
1347 + .pipe(Effect.exit);
1348 + assert.strictEqual(result._tag, "Failure");
1349 + assert.deepStrictEqual(recovery.calls, []);
1350 + }).pipe(Effect.scoped, Effect.provide(testLayer)),
1351 + );
1352 +
1353 it.effect("reports a no-op compaction as a visible failure and permits another turn", () =>
1354 Effect.gen(function* () {
1355 const fake = yield* makeFakeMuse();
packages/provider-muse/src/server/adapter.ts
+20 -4
@@ -324,6 +324,8 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
324 let hostEpoch = 0;
325 let closed = false;
326 let broken = false;
327 + // Only idle host failures can be retried on this runtime.
328 + let relaunchable = false;
329 let nativeSessionId: string | undefined;
330 let thread: OrchestrationV2ProviderThread | undefined;
331 let active: ActiveTurn | undefined;
@@ -1035,7 +1037,13 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
1037 });
1038 const failHost = Effect.fnUntraced(function* (cause: unknown) {
1039 if (closed || broken) return;
1040 + hostEpoch++;
1041 + // Registration can replace the host while failure events are consumed.
1042 + const failed = host;
1043 + const turn = active;
1044 broken = true;
1045 + relaunchable = !turn;
1046 + wake = undefined;
1047 const rootCause = Cause.isCause(cause) ? Cause.squash(cause) : cause;
1048 const failure = isOpenSessionError(rootCause) ? rootCause.cause : rootCause;
1049 const detail = isProtocolError(failure)
@@ -1043,14 +1051,13 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
1051 : failure instanceof Error
1052 ? failure.message
1053 : "Muse transport failed.";
1046 - if (active) yield* finish(active, "failed", detail, "broken");
1054 + if (turn) yield* finish(turn, "failed", detail, "broken");
1055 else {
1056 yield* updateThread({ status: "error" });
1057 yield* updateSession("error", detail);
1058 }
1059 yield* settleObservedChildren("failed");
1052 - yield* Queue.fail(events, protocolError(detail, cause));
1053 - const failed = host;
1060 + if (turn) yield* Queue.fail(events, protocolError(detail, cause));
1061 yield* Effect.tryPromise(() => failed.close()).pipe(Effect.ignore, Effect.forkIn(scope));
1062 });
1063 const owns = (turn: ActiveTurn, turnId: string) =>
@@ -1736,8 +1743,17 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
1743 const register = Effect.fnUntraced(function* (
1744 args: ProviderAdapter.ProviderAdapterV2EnsureThreadInput,
1745 ) {
1739 - if (broken || closed)
1746 + if (closed || (broken && !relaunchable))
1747 return yield* protocolError("Muse host is unavailable; reopen the session");
1748 + if (broken) {
1749 + yield* Effect.logWarning("Relaunching the Muse host after a failure");
1750 + yield* launchHost();
1751 + broken = false;
1752 + relaunchable = false;
1753 + thread = undefined;
1754 + nativeSessionId = undefined;
1755 + yield* updateSession("ready", null);
1756 + }
1757 if (active)
1758 return yield* protocolError("Cannot change Muse sessions during an active turn");
1759 const existing = args.existingProviderThread;