The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 23:15 UTC
fix(provider-muse): relaunch a host that died between turns (#19195)
Co-authored-by: dan <dan@bw-hq-linuxws01.bellwetherit.com> Co-authored-by: Julius Marminge <julius0216@outlook.com>
Dan Donathan committed
Oct 19, 2026 at 22:03 UTC
4bf8b476fa749708b7c1343ba9c9959edb2de8b9
4 files changed
+318
-8
apps/server/src/orchestration-v2/ProviderSessionManager.test.ts
+121
@@ -4446,6 +4446,127 @@ it.effect(
4446
}),
4447
);
4448
4449
+it.effect(
4450
+ "ProviderSessionManagerV2 invalidates only errored threads and keeps reusable session errors cached",
4451
+ () =>
4452
+ Effect.gen(function* () {
4453
+ const state = yield* Ref.make(emptyState);
4454
+ const effect = Effect.gen(function* () {
4455
+ const eventSink = yield* EventSink.EventSinkV2;
4456
+ const idAllocator = yield* IdAllocator.IdAllocatorV2;
4457
+ const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
4458
+ const now = yield* DateTime.now;
4459
+ const openCachedRuntime = Effect.fnUntraced(function* (
4460
+ threadId: ThreadId,
4461
+ sharedSessionId?: ProviderSessionId,
4462
+ ) {
4463
+ const providerSessionId =
4464
+ sharedSessionId ??
4465
+ (yield* idAllocator.allocate.providerSession({
4466
+ providerInstanceId: modelSelection.instanceId,
4467
+ threadId,
4468
+ }));
4469
+ yield* eventSink.write({
4470
+ events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })],
4471
+ });
4472
+ const runtime = yield* manager.open({
4473
+ threadId,
4474
+ providerSessionId,
4475
+ modelSelection,
4476
+ runtimePolicy,
4477
+ });
4478
+ const providerThread = makeProviderThread({
4479
+ idAllocator,
4480
+ threadId,
4481
+ providerSessionId,
4482
+ now,
4483
+ nativeThreadId: String(threadId),
4484
+ });
4485
+ const resume = runtime.resumeThread({
4486
+ providerThread,
4487
+ threadId,
4488
+ modelSelection,
4489
+ runtimePolicy,
4490
+ });
4491
+ yield* resume;
4492
+ yield* resume;
4493
+ return { runtime, providerThread, resume };
4494
+ });
4495
+ const failed = yield* openCachedRuntime(ThreadId.make("idle-failed-runtime"));
4496
+ const healthy = yield* openCachedRuntime(ThreadId.make("idle-healthy-runtime"));
4497
+ const sibling = yield* openCachedRuntime(
4498
+ ThreadId.make("idle-healthy-sibling"),
4499
+ failed.runtime.providerSessionId,
4500
+ );
4501
+ assert.strictEqual(sibling.runtime, failed.runtime);
4502
+ assert.equal((yield* Ref.get(state)).openCount, 2);
4503
+ assert.equal((yield* Ref.get(state)).resumeCount, 3);
4504
+
4505
+ const queue = (yield* Ref.get(state)).eventQueues.get(
4506
+ String(failed.runtime.providerSessionId),
4507
+ );
4508
+ assert.isDefined(queue);
4509
+ const publish = Effect.fnUntraced(function* (
4510
+ event: ProviderAdapter.ProviderAdapterV2Event,
4511
+ ) {
4512
+ const subscription = yield* failed.runtime.subscribeEvents!;
4513
+ yield* Queue.offer(queue!, event);
4514
+ // Observing the exact event ensures the manager has handled it before resuming.
4515
+ assert.deepEqual(yield* subscription.events.pipe(Stream.runHead), Option.some(event));
4516
+ });
4517
+ const countRawResumes = Effect.fnUntraced(function* (cached: typeof failed) {
4518
+ const before = (yield* Ref.get(state)).resumeCount;
4519
+ yield* cached.resume;
4520
+ return (yield* Ref.get(state)).resumeCount - before;
4521
+ });
4522
+ yield* publish({
4523
+ type: "provider_thread.updated",
4524
+ driver: CODEX_DRIVER,
4525
+ providerThread: failed.providerThread,
4526
+ });
4527
+ yield* publish({
4528
+ type: "provider_session.updated",
4529
+ driver: CODEX_DRIVER,
4530
+ providerSession: {
4531
+ ...failed.runtime.providerSession,
4532
+ status: "error",
4533
+ lastError: "Reusable turn failure",
4534
+ },
4535
+ });
4536
+ const afterSessionError = {
4537
+ reusableThread: yield* countRawResumes(failed),
4538
+ siblingThread: yield* countRawResumes(sibling),
4539
+ };
4540
+
4541
+ yield* publish({
4542
+ type: "provider_thread.updated",
4543
+ driver: CODEX_DRIVER,
4544
+ providerThread: { ...failed.providerThread, status: "error" },
4545
+ });
4546
+ const afterThreadError = {
4547
+ healthyRuntime: yield* countRawResumes(healthy),
4548
+ siblingThread: yield* countRawResumes(sibling),
4549
+ failedThread: yield* countRawResumes(failed),
4550
+ failedThreadAgain: yield* countRawResumes(failed),
4551
+ };
4552
+ assert.deepEqual(
4553
+ { afterSessionError, afterThreadError },
4554
+ {
4555
+ afterSessionError: { reusableThread: 0, siblingThread: 0 },
4556
+ afterThreadError: {
4557
+ healthyRuntime: 0,
4558
+ siblingThread: 0,
4559
+ failedThread: 1,
4560
+ failedThreadAgain: 0,
4561
+ },
4562
+ },
4563
+ );
4564
+ });
4565
+
4566
+ yield* effect.pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 1000 })));
4567
+ }),
4568
+);
4569
+
4570
it.effect(
4571
"ProviderSessionManagerV2 re-attaching a thread waits for its in-flight unload, then reloads it",
4572
() =>
apps/server/src/orchestration-v2/ProviderSessionManager.ts
+19
@@ -1969,6 +1969,25 @@ export const layerWithOptions = (
1969
),
1970
Effect.andThen(
1971
Effect.gen(function* () {
1972
+ if (
1973
+ event.type === "provider_thread.updated" &&
1974
+ event.providerThread.status === "error"
1975
+ ) {
1976
+ const threadId = event.providerThread.appThreadId;
1977
+ yield* Ref.update(sessions, (sessions) => {
1978
+ const key = sessionKey(entry.runtime.providerSessionId);
1979
+ const latest = sessions.get(key);
1980
+ if (latest?.runtime !== entry.runtime || threadId === null) return sessions;
1981
+ const loadedProviderThreadKeyByThread = new Map(
1982
+ latest.loadedProviderThreadKeyByThread,
1983
+ );
1984
+ loadedProviderThreadKeyByThread.delete(threadId);
1985
+ return new Map(sessions).set(key, {
1986
+ ...latest,
1987
+ loadedProviderThreadKeyByThread,
1988
+ });
1989
+ });
1990
+ }
1991
// Some providers can block before a run subscriber exists
1992
// (project trust, login, or session-switch hooks). Persist
1993
// their runless request artifacts directly so the normal T3
packages/provider-muse/src/server/adapter.test.ts
+158
-4
@@ -832,7 +832,25 @@ describe("MuseAdapterV2", () => {
832
Effect.gen(function* () {
833
const fake = yield* makeFakeMuse();
834
const harness = yield* makeHarness(fake);
835
- const { nativeId } = yield* startConversation(harness, fake);
835
+ const first = yield* startConversation(harness, fake);
836
+ yield* fake.emit("item/started", {
837
+ turnId: first.nativeId,
838
+ item: {
839
+ itemId: "background-child",
840
+ kind: "subagent",
841
+ revision: 1,
842
+ status: "inProgress",
843
+ turnId: first.nativeId,
844
+ subagentId: "subagent-1",
845
+ controlStatus: "running",
846
+ childSessionId: "child-session-1",
847
+ objective: "Finish in the background",
848
+ },
849
+ });
850
+ const child = yield* harness.takeEvent("app_thread.created");
851
+ yield* fake.emit("turn/completed", { turnId: first.nativeId, terminal: "completed" });
852
+ yield* harness.takeEvent("turn.terminal");
853
+ const { nativeId } = yield* startConversation(harness, fake, 2);
854
yield* fake.emit("approval/requested", approval(nativeId));
855
yield* harness.takeEvent(
856
"runtime_request.updated",
@@ -848,15 +866,36 @@ describe("MuseAdapterV2", () => {
866
event.type === "runtime_request.updated" && event.runtimeRequest.status === "cancelled",
867
),
868
);
869
+ const streamExit = yield* Deferred.await(harness.eventsEnded);
870
+ assert.strictEqual(streamExit._tag, "Failure");
871
+ assert.isTrue(
872
+ harness.allEvents.some(
873
+ (event) =>
874
+ event.type === "subagent.updated" &&
875
+ event.subagent.childThreadId === child.appThread.id &&
876
+ event.subagent.status === "failed",
877
+ ),
878
+ );
879
+ assert.isTrue(
880
+ harness.allEvents.some(
881
+ (event) =>
882
+ event.type === "node.updated" &&
883
+ event.node.threadId === child.appThread.id &&
884
+ event.node.kind === "root_turn" &&
885
+ event.node.status === "failed",
886
+ ),
887
+ );
888
const result = yield* harness.runtime
852
- .startTurn(yield* turnInput(harness.providerThread, 2))
889
+ .startTurn(yield* turnInput(harness.providerThread, 3))
890
.pipe(Effect.exit);
891
assert.strictEqual(result._tag, "Failure");
892
assert.strictEqual(
856
- harness.allEvents.filter((event) => event.type === "turn.terminal").length,
893
+ harness.allEvents.filter(
894
+ (event) => event.type === "turn.terminal" && event.runOrdinal === 2,
895
+ ).length,
896
1,
897
);
859
- assert.strictEqual(fake.calls.filter((call) => call.method === "turn/start").length, 1);
898
+ assert.strictEqual(fake.calls.filter((call) => call.method === "turn/start").length, 2);
899
}).pipe(Effect.scoped, Effect.provide(testLayer)),
900
);
901
@@ -1167,6 +1206,92 @@ describe("MuseAdapterV2", () => {
1206
}).pipe(Effect.scoped, Effect.provide(testLayer)),
1207
);
1208
1209
+ it.effect.each([
1210
+ { resume: true, label: "resumes the saved session" },
1211
+ { resume: false, label: "starts a fresh session" },
1212
+ ])("completes a turn after an idle host dies and $label", ({ resume }) =>
1213
+ Effect.gen(function* () {
1214
+ const fake = yield* makeFakeMuse("first");
1215
+ const recovery = yield* makeFakeMuse("second");
1216
+ const harness = yield* makeHarness(fake, INSTANCE_ID, "saved-session", recovery);
1217
+ yield* fake.disconnect();
1218
+ yield* harness.takeEvent(
1219
+ "provider_session.updated",
1220
+ (event) => event.providerSession.status === "error",
1221
+ );
1222
+
1223
+ const reEnsured = yield* harness.runtime.ensureThread({
1224
+ threadId: THREAD_ID,
1225
+ modelSelection: modelSelection(),
1226
+ runtimePolicy,
1227
+ existingProviderThread: resume
1228
+ ? harness.providerThread
1229
+ : { ...harness.providerThread, nativeThreadRef: null },
1230
+ });
1231
+ const registered = yield* recovery.takeCall(resume ? "session/resume" : "session/start");
1232
+ assert.strictEqual(registered.params.sessionId, reEnsured.nativeThreadRef?.nativeId);
1233
+ assert.strictEqual(reEnsured.id, harness.providerThread.id);
1234
+ assert.strictEqual(reEnsured.nativeThreadRef?.nativeId === "saved-session", resume);
1235
+
1236
+ yield* harness.runtime.startTurn(yield* turnInput(reEnsured));
1237
+ const started = yield* recovery.takeCall("turn/start");
1238
+ yield* recovery.emit("turn/completed", {
1239
+ turnId: started.commandId,
1240
+ terminal: "completed",
1241
+ });
1242
+ const terminal = yield* harness.takeEvent("turn.terminal");
1243
+ assert.strictEqual(terminal.status, "completed");
1244
+ assert.strictEqual(terminal.providerThreadId, reEnsured.id);
1245
+ assert.strictEqual(terminal.threadDisposition, "reusable");
1246
+ }).pipe(Effect.scoped, Effect.provide(testLayer)),
1247
+ );
1248
+
1249
+ it.effect("discards a dead host's pending continuation before relaunching", () =>
1250
+ Effect.gen(function* () {
1251
+ const fake = yield* makeFakeMuse("first");
1252
+ const recovery = yield* makeFakeMuse("second");
1253
+ const offered =
1254
+ yield* Deferred.make<ProviderContinuationRequests.ProviderContinuationRequest>();
1255
+ const harness = yield* makeHarness(
1256
+ fake,
1257
+ INSTANCE_ID,
1258
+ "saved-session",
1259
+ recovery,
1260
+ undefined,
1261
+ runtimePolicy,
1262
+ { continuationRequests: { offer: (request) => Deferred.succeed(offered, request) } },
1263
+ );
1264
+ yield* fake.emit("turn/started", { turnId: "dead-host-report" });
1265
+ const continuation = yield* Deferred.await(offered);
1266
+ assert.isTrue(yield* harness.runtime.hasPendingBackgroundWork!);
1267
+ yield* fake.disconnect();
1268
+ yield* harness.takeEvent(
1269
+ "provider_session.updated",
1270
+ (event) => event.providerSession.status === "error",
1271
+ );
1272
+ assert.isFalse(yield* harness.runtime.hasPendingBackgroundWork!);
1273
+ const dispatched = yield* continuation.dispatchIfCurrent!(
1274
+ Effect.die("Stale continuation dispatched"),
1275
+ );
1276
+ assert.isTrue(Option.isNone(dispatched));
1277
+
1278
+ const reEnsured = yield* harness.runtime.ensureThread({
1279
+ threadId: THREAD_ID,
1280
+ modelSelection: modelSelection(),
1281
+ runtimePolicy,
1282
+ existingProviderThread: harness.providerThread,
1283
+ });
1284
+ const input = yield* turnInput(reEnsured);
1285
+ yield* harness.runtime.startTurn({
1286
+ ...input,
1287
+ message: { ...input.message, createdBy: "agent", creationSource: "provider" },
1288
+ });
1289
+ const running = yield* harness.takeEvent("provider_turn.updated");
1290
+ assert.notStrictEqual(running.providerTurn.nativeTurnRef?.nativeId, "dead-host-report");
1291
+ assert.strictEqual((yield* harness.takeEvent("turn.terminal")).status, "completed");
1292
+ }).pipe(Effect.scoped, Effect.provide(testLayer)),
1293
+ );
1294
+
1295
it.effect("serializes interruption behind pending turn admission without losing the target", () =>
1296
Effect.gen(function* () {
1297
const fake = yield* makeFakeMuse();
@@ -1196,6 +1321,35 @@ describe("MuseAdapterV2", () => {
1321
}).pipe(Effect.scoped, Effect.provide(testLayer)),
1322
);
1323
1324
+ it.effect("keeps an intentionally interrupted compaction closed on ensureThread", () =>
1325
+ Effect.gen(function* () {
1326
+ const fake = yield* makeFakeMuse();
1327
+ const recovery = yield* makeFakeMuse("replacement");
1328
+ const harness = yield* makeHarness(fake, INSTANCE_ID, undefined, recovery);
1329
+ yield* harness.runtime.compactThread!(yield* turnInput(harness.providerThread));
1330
+ const running = yield* harness.takeEvent("provider_turn.updated");
1331
+ yield* harness.runtime.interruptTurn({
1332
+ providerThread: harness.providerThread,
1333
+ providerTurnId: running.providerTurn.id,
1334
+ });
1335
+ const terminal = yield* harness.takeEvent("turn.terminal");
1336
+ assert.strictEqual(terminal.status, "interrupted");
1337
+ assert.strictEqual(terminal.threadDisposition, "broken");
1338
+ assert.strictEqual((yield* Deferred.await(harness.eventsEnded))._tag, "Success");
1339
+
1340
+ const result = yield* harness.runtime
1341
+ .ensureThread({
1342
+ threadId: THREAD_ID,
1343
+ modelSelection: modelSelection(),
1344
+ runtimePolicy,
1345
+ existingProviderThread: harness.providerThread,
1346
+ })
1347
+ .pipe(Effect.exit);
1348
+ assert.strictEqual(result._tag, "Failure");
1349
+ assert.deepStrictEqual(recovery.calls, []);
1350
+ }).pipe(Effect.scoped, Effect.provide(testLayer)),
1351
+ );
1352
+
1353
it.effect("reports a no-op compaction as a visible failure and permits another turn", () =>
1354
Effect.gen(function* () {
1355
const fake = yield* makeFakeMuse();
packages/provider-muse/src/server/adapter.ts
+20
-4
@@ -324,6 +324,8 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
324
let hostEpoch = 0;
325
let closed = false;
326
let broken = false;
327
+ // Only idle host failures can be retried on this runtime.
328
+ let relaunchable = false;
329
let nativeSessionId: string | undefined;
330
let thread: OrchestrationV2ProviderThread | undefined;
331
let active: ActiveTurn | undefined;
@@ -1035,7 +1037,13 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
1037
});
1038
const failHost = Effect.fnUntraced(function* (cause: unknown) {
1039
if (closed || broken) return;
1040
+ hostEpoch++;
1041
+ // Registration can replace the host while failure events are consumed.
1042
+ const failed = host;
1043
+ const turn = active;
1044
broken = true;
1045
+ relaunchable = !turn;
1046
+ wake = undefined;
1047
const rootCause = Cause.isCause(cause) ? Cause.squash(cause) : cause;
1048
const failure = isOpenSessionError(rootCause) ? rootCause.cause : rootCause;
1049
const detail = isProtocolError(failure)
@@ -1043,14 +1051,13 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
1051
: failure instanceof Error
1052
? failure.message
1053
: "Muse transport failed.";
1046
- if (active) yield* finish(active, "failed", detail, "broken");
1054
+ if (turn) yield* finish(turn, "failed", detail, "broken");
1055
else {
1056
yield* updateThread({ status: "error" });
1057
yield* updateSession("error", detail);
1058
}
1059
yield* settleObservedChildren("failed");
1052
- yield* Queue.fail(events, protocolError(detail, cause));
1053
- const failed = host;
1060
+ if (turn) yield* Queue.fail(events, protocolError(detail, cause));
1061
yield* Effect.tryPromise(() => failed.close()).pipe(Effect.ignore, Effect.forkIn(scope));
1062
});
1063
const owns = (turn: ActiveTurn, turnId: string) =>
@@ -1736,8 +1743,17 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
1743
const register = Effect.fnUntraced(function* (
1744
args: ProviderAdapter.ProviderAdapterV2EnsureThreadInput,
1745
) {
1739
- if (broken || closed)
1746
+ if (closed || (broken && !relaunchable))
1747
return yield* protocolError("Muse host is unavailable; reopen the session");
1748
+ if (broken) {
1749
+ yield* Effect.logWarning("Relaunching the Muse host after a failure");
1750
+ yield* launchHost();
1751
+ broken = false;
1752
+ relaunchable = false;
1753
+ thread = undefined;
1754
+ nativeSessionId = undefined;
1755
+ yield* updateSession("ready", null);
1756
+ }
1757
if (active)
1758
return yield* protocolError("Cannot change Muse sessions during an active turn");
1759
const existing = args.existingProviderThread;