The best way to control Claude Code, Codex and more from any computer or phone
fix(muse): retries show while Muse waits and fold away once it recovers (#33151)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Theo Browne committed
Nov 8, 2026 at 17:58 UTC
7da0ef0e1f874caca679bab15a309b9542aecc44
3 files changed
+187
-18
packages/provider-muse/src/server/adapter.test.ts
+83
@@ -1888,6 +1888,89 @@ describe("MuseAdapterV2", () => {
1888
}).pipe(Effect.scoped, Effect.provide(testLayer)),
1889
);
1890
1891
+ it.effect("keeps each retry chain on one row that settles when the turn moves on", () =>
1892
+ Effect.gen(function* () {
1893
+ const fake = yield* makeFakeMuse();
1894
+ const harness = yield* makeHarness(fake);
1895
+ const { nativeId } = yield* startConversation(harness, fake);
1896
+ const takeRetryRow = Effect.fnUntraced(function* () {
1897
+ const { turnItem } = yield* harness.takeEvent(
1898
+ "turn_item.updated",
1899
+ (event) => event.turnItem.type === "error",
1900
+ );
1901
+ return turnItem.type === "error" ? turnItem : undefined;
1902
+ });
1903
+ const retry = Effect.fnUntraced(function* (
1904
+ method: string,
1905
+ record: string,
1906
+ attempt: number,
1907
+ params: Record<string, unknown>,
1908
+ ) {
1909
+ yield* fake.emit(method, {
1910
+ turnId: nativeId,
1911
+ attempt,
1912
+ nextAttempt: attempt + 1,
1913
+ sourceRange: { first: { id: record }, last: { id: record } },
1914
+ ...params,
1915
+ });
1916
+ return yield* takeRetryRow();
1917
+ });
1918
+ const timeout = { maxAttempts: 2, retryDelayMs: 1_000, reason: "no first model event" };
1919
+ const turnRetry = yield* retry("turn/retryScheduled", "record-1", 1, timeout);
1920
+ assert.strictEqual(turnRetry?.status, "running");
1921
+ assert.strictEqual(turnRetry?.failure.message, "no first model event. Retrying in 1s.");
1922
+ assert.deepStrictEqual(turnRetry?.retry, { attempt: 1, maxAttempts: 1, retryDelayMs: 1_000 });
1923
+ const http = { maxAttempts: 10, errorKind: "server", httpStatus: 504, retryDelayMs: 1_000 };
1924
+ const modelRetry = yield* retry("turn/modelRetryScheduled", "record-2", 1, http);
1925
+ assert.strictEqual(
1926
+ modelRetry?.failure.message,
1927
+ "Model request failed (HTTP 504). Retrying in 1s.",
1928
+ );
1929
+ const nextChain = yield* retry("turn/retryScheduled", "record-3", 1, timeout);
1930
+ assert.strictEqual(new Set([turnRetry?.id, modelRetry?.id, nextChain?.id]).size, 3);
1931
+ const sameChain = yield* retry("turn/modelRetryScheduled", "record-4", 2, {
1932
+ ...http,
1933
+ httpStatus: 429,
1934
+ retryDelayMs: 60_000,
1935
+ });
1936
+ assert.strictEqual(sameChain?.id, modelRetry?.id);
1937
+ assert.deepStrictEqual(sameChain?.retry, {
1938
+ attempt: 2,
1939
+ maxAttempts: 9,
1940
+ retryDelayMs: 60_000,
1941
+ });
1942
+
1943
+ yield* fake.emit("item/started", {
1944
+ turnId: nativeId,
1945
+ item: {
1946
+ itemId: "answer",
1947
+ turnId: nativeId,
1948
+ kind: "agentMessage",
1949
+ revision: 1,
1950
+ status: "inProgress",
1951
+ },
1952
+ });
1953
+ const recovered = [yield* takeRetryRow(), yield* takeRetryRow(), yield* takeRetryRow()];
1954
+ assert.sameMembers(
1955
+ recovered.map((row) => `${row?.id} ${row?.status} ${row?.failure.message}`),
1956
+ [
1957
+ `${turnRetry?.id} completed no first model event`,
1958
+ `${nextChain?.id} completed no first model event`,
1959
+ `${sameChain?.id} completed Model request failed (HTTP 429)`,
1960
+ ],
1961
+ );
1962
+
1963
+ const stalled = yield* retry("turn/modelRetryScheduled", "record-5", 1, http);
1964
+ yield* fake.emit("turn/completed", {
1965
+ turnId: nativeId,
1966
+ terminal: "failed",
1967
+ error: { message: "Muse gave up" },
1968
+ });
1969
+ const stopped = yield* takeRetryRow();
1970
+ assert.deepStrictEqual([stopped?.id, stopped?.status], [stalled?.id, "interrupted"]);
1971
+ }).pipe(Effect.scoped, Effect.provide(testLayer)),
1972
+ );
1973
+
1974
it.effect("opens a prompt-and-result child thread for a native subagent", () =>
1975
Effect.gen(function* () {
1976
const fake = yield* makeFakeMuse();
packages/provider-muse/src/server/adapter.ts
+97
-16
@@ -7,6 +7,7 @@ import {
7
type OrchestrationV2ConversationMessage,
8
type OrchestrationV2ExecutionNode,
9
type OrchestrationV2ProviderCapabilities,
10
+ type OrchestrationV2ProviderRetry,
11
type OrchestrationV2ProviderSession,
12
type OrchestrationV2ProviderThread,
13
type OrchestrationV2ProviderTurn,
@@ -26,6 +27,7 @@ import {
27
import type { MuseSettings } from "../settings.ts";
28
import { AgentScope } from "@t3tools/shared/AgentScope";
29
import { getModelSelectionStringOptionValue } from "@t3tools/shared/model";
30
+import { formatDuration } from "@t3tools/shared/orchestrationTiming";
31
import * as Cause from "effect/Cause";
32
import * as DateTime from "effect/DateTime";
33
import * as Deferred from "effect/Deferred";
@@ -52,7 +54,7 @@ import {
54
MuseSubscriptionUsage,
55
MuseTokenUsageEvent,
56
MuseTodoList,
55
- MuseTurnRetryScheduled,
57
+ MuseRetryScheduled,
58
MuseTurnCompleted,
59
MuseTurnStartResult,
60
MuseUserInput,
@@ -84,7 +86,10 @@ import {
86
type BackgroundWorkReport,
87
} from "@t3tools/provider-core/server/notification";
88
import type * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests";
87
-import { makeProviderFailure } from "@t3tools/provider-core/server/failure";
89
+import {
90
+ makeProviderFailure,
91
+ makeProviderRetryTurnItem,
92
+} from "@t3tools/provider-core/server/failure";
93
import {
94
makeSubagentChildThread,
95
makeSubagentConversationArtifacts,
@@ -214,6 +219,7 @@ interface ActiveTurn {
219
readonly started: Map<string, DateTime.Utc>;
220
readonly dirty: Set<string>;
221
readonly settledRequests: Set<string>;
222
+ readonly retries: Array<MuseRetryRow>;
223
readonly done: Deferred.Deferred<void>;
224
todoPlanId?: PlanId;
225
nextOrdinal: number;
@@ -222,6 +228,15 @@ interface ActiveTurn {
228
compact: boolean;
229
}
230
231
+interface MuseRetryRow {
232
+ readonly method: string;
233
+ readonly nativeId: string;
234
+ readonly retry: OrchestrationV2ProviderRetry;
235
+ readonly cause: string;
236
+ readonly code: string | undefined;
237
+ running: boolean;
238
+}
239
+
240
interface PendingRequest {
241
readonly native:
242
| { type: "approval"; value: MuseApproval }
@@ -258,9 +273,12 @@ const INFORMATIONAL_NOTIFICATIONS = new Set([
273
"session/tokenUsage",
274
"session/todoListChanged",
275
"turn/retryScheduled",
276
+ "turn/modelRetryScheduled",
277
"usage/changed",
278
]);
279
const TOOLS_WITH_NATIVE_ROWS = new Set(["write_todos", "request_user_input", "workflow"]);
280
+// Reminder children and other runtime items can start while the model is still retrying.
281
+const MODEL_OUTPUT_KINDS = new Set(["agentMessage", "reasoning", "toolCall"]);
282
const responseAnswerSchema = Schema.Union([Schema.String, Schema.Array(Schema.String)]);
283
284
export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
@@ -447,6 +465,52 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
465
updatedAt: time,
466
};
467
};
468
+ const emitRetry = Effect.fnUntraced(function* (
469
+ turn: ActiveTurn,
470
+ row: MuseRetryRow,
471
+ status: "running" | "completed" | "interrupted",
472
+ ) {
473
+ const time = yield* DateTime.now;
474
+ const base = baseItem(turn, row.nativeId, time);
475
+ yield* emit({
476
+ type: "turn_item.updated",
477
+ driver: MUSE_PROVIDER,
478
+ turnItem: {
479
+ ...makeProviderRetryTurnItem({
480
+ ...base,
481
+ driver: MUSE_PROVIDER,
482
+ nodeId: turn.input.rootNodeId,
483
+ itemOrdinal: base.ordinal,
484
+ failure: makeProviderFailure({
485
+ // Clients show only the failure message, so a waiting row carries its delay.
486
+ message:
487
+ status === "running"
488
+ ? `${row.cause}. Retrying in ${formatDuration(row.retry.retryDelayMs ?? 0)}.`
489
+ : row.cause,
490
+ code: row.code,
491
+ class: "provider_error",
492
+ retryable: true,
493
+ }),
494
+ retry: row.retry,
495
+ status,
496
+ updatedAt: time,
497
+ }),
498
+ // The shared helper keys one row per turn; a long Muse turn can retry several times.
499
+ id: base.id,
500
+ },
501
+ });
502
+ });
503
+ // Muse sends no recovered event, so model output or the turn's end closes open retry rows.
504
+ const settleRetries = Effect.fnUntraced(function* (
505
+ turn: ActiveTurn,
506
+ status: "completed" | "interrupted",
507
+ ) {
508
+ for (const row of turn.retries) {
509
+ if (!row.running) continue;
510
+ row.running = false;
511
+ yield* emitRetry(turn, row, status);
512
+ }
513
+ });
514
const childItemIdentity = (context: ChildContext, nativeId: string) =>
515
`${options.instanceId}:child:${context.childSessionId}:${nativeId}`;
516
const itemStatus = (item: MuseItem): OrchestrationV2TurnItemStatus => {
@@ -970,6 +1034,8 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
1034
yield* publishItem(turn, item, item.status === "inProgress" ? status : undefined);
1035
}
1036
turn.dirty.clear();
1037
+ // A failed turn has its own failure row; its retry rows only record that retrying stopped.
1038
+ yield* settleRetries(turn, status === "completed" ? "completed" : "interrupted");
1039
for (const entry of pending.values())
1040
if (entry.request.providerTurnId !== null) yield* resolvePending(entry, "cancelled");
1041
turn.providerTurn = { ...turn.providerTurn, status, completedAt };
@@ -1413,6 +1479,8 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
1479
return;
1480
const previous = turn.items.get(item.itemId);
1481
if (previous && previous.revision >= item.revision) return;
1482
+ if (method === "item/started" && MODEL_OUTPUT_KINDS.has(item.kind))
1483
+ yield* settleRetries(turn, "completed");
1484
turn.items.set(item.itemId, item);
1485
turn.dirty.delete(item.itemId);
1486
yield* publishItem(turn, item);
@@ -1507,6 +1575,7 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
1575
break;
1576
case "session/tokenUsage": {
1577
const usage = yield* decode(MuseTokenUsageEvent, params);
1578
+ yield* settleRetries(turn, "completed");
1579
const previous = turn.providerTurn.turnTokenUsage;
1580
const contextUsage = thread?.contextUsage;
1581
turn.providerTurn = {
@@ -1630,21 +1699,32 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
1699
});
1700
break;
1701
}
1633
- case "turn/retryScheduled": {
1634
- const retry = yield* decode(MuseTurnRetryScheduled, params);
1635
- const time = yield* DateTime.now;
1636
- yield* emit({
1637
- type: "turn_item.updated",
1638
- driver: MUSE_PROVIDER,
1639
- turnItem: {
1640
- ...baseItem(turn, `retry:${retry.nextAttempt}`, time),
1641
- type: "system_notice",
1642
- status: "completed",
1643
- title: "Muse retry",
1644
- completedAt: time,
1645
- message: `Muse is retrying this turn (attempt ${retry.nextAttempt} of ${retry.maxAttempts}): ${retry.reason}`,
1702
+ case "turn/retryScheduled":
1703
+ case "turn/modelRetryScheduled": {
1704
+ const retry = yield* decode(MuseRetryScheduled, params);
1705
+ const cause =
1706
+ retry.httpStatus === undefined ? retry.errorKind : `HTTP ${retry.httpStatus}`;
1707
+ // A chain's first retry opens a row; its later attempts update that row.
1708
+ const index =
1709
+ retry.attempt > 1 ? turn.retries.findLastIndex((row) => row.method === method) : -1;
1710
+ const row: MuseRetryRow = {
1711
+ method,
1712
+ nativeId: turn.retries[index]?.nativeId ?? `retry:${retry.sourceRange.first.id}`,
1713
+ // Muse counts the first request as attempt 1; T3 counts retries.
1714
+ retry: {
1715
+ attempt: retry.attempt,
1716
+ maxAttempts: Math.max(1, retry.maxAttempts - 1),
1717
+ retryDelayMs: Math.trunc(retry.retryDelayMs),
1718
},
1647
- });
1719
+ cause:
1720
+ retry.reason ??
1721
+ (cause ? `Model request failed (${cause})` : "Model request failed"),
1722
+ code: retry.errorKind,
1723
+ running: true,
1724
+ };
1725
+ if (index === -1) turn.retries.push(row);
1726
+ else turn.retries[index] = row;
1727
+ yield* emitRetry(turn, row, "running");
1728
break;
1729
}
1730
}
@@ -2041,6 +2121,7 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
2121
started: new Map(),
2122
dirty: new Set(),
2123
settledRequests: new Set(),
2124
+ retries: [],
2125
done: yield* Deferred.make<void>(),
2126
nextOrdinal: turnInput.providerTurnOrdinal * 100 + 1,
2127
flushScheduled: false,
packages/provider-muse/src/server/protocol.ts
+7
-2
@@ -182,12 +182,17 @@ export const MuseSubscriptionUsage = Schema.Struct({
182
}),
183
});
184
export type MuseSubscriptionUsage = typeof MuseSubscriptionUsage.Type;
185
-export const MuseTurnRetryScheduled = Schema.Struct({
185
+/** `turn/retryScheduled` carries `reason`; `turn/modelRetryScheduled` carries the HTTP failure. */
186
+export const MuseRetryScheduled = Schema.Struct({
187
turnId: NonEmptyString,
188
+ attempt: Schema.Int.check(Schema.isGreaterThan(0)),
189
nextAttempt: Schema.Int.check(Schema.isGreaterThan(0)),
190
maxAttempts: Schema.Int.check(Schema.isGreaterThan(0)),
189
- reason: NonEmptyString,
191
+ reason: Schema.optional(NonEmptyString),
192
+ errorKind: Schema.optional(Schema.String),
193
+ httpStatus: Schema.optional(Schema.Int),
194
retryDelayMs: Schema.Finite.check(Schema.isGreaterThanOrEqualTo(0)),
195
+ sourceRange: Schema.Struct({ first: Schema.Struct({ id: NonEmptyString }) }),
196
});
197
export const MuseCompactResult = Schema.Struct({
198
status: Schema.String,