@devries-cameron20 / t3code-1 / commits / 7da0ef0e1f
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(muse): retries show while Muse waits and fold away once it recovers (#33151)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Theo Browne committed Nov 8, 2026 at 17:58 UTC 7da0ef0e1f874caca679bab15a309b9542aecc44
3 files changed +187 -18
packages/provider-muse/src/server/adapter.test.ts
+83
@@ -1888,6 +1888,89 @@ describe("MuseAdapterV2", () => {
1888 }).pipe(Effect.scoped, Effect.provide(testLayer)),
1889 );
1890
1891 + it.effect("keeps each retry chain on one row that settles when the turn moves on", () =>
1892 + Effect.gen(function* () {
1893 + const fake = yield* makeFakeMuse();
1894 + const harness = yield* makeHarness(fake);
1895 + const { nativeId } = yield* startConversation(harness, fake);
1896 + const takeRetryRow = Effect.fnUntraced(function* () {
1897 + const { turnItem } = yield* harness.takeEvent(
1898 + "turn_item.updated",
1899 + (event) => event.turnItem.type === "error",
1900 + );
1901 + return turnItem.type === "error" ? turnItem : undefined;
1902 + });
1903 + const retry = Effect.fnUntraced(function* (
1904 + method: string,
1905 + record: string,
1906 + attempt: number,
1907 + params: Record<string, unknown>,
1908 + ) {
1909 + yield* fake.emit(method, {
1910 + turnId: nativeId,
1911 + attempt,
1912 + nextAttempt: attempt + 1,
1913 + sourceRange: { first: { id: record }, last: { id: record } },
1914 + ...params,
1915 + });
1916 + return yield* takeRetryRow();
1917 + });
1918 + const timeout = { maxAttempts: 2, retryDelayMs: 1_000, reason: "no first model event" };
1919 + const turnRetry = yield* retry("turn/retryScheduled", "record-1", 1, timeout);
1920 + assert.strictEqual(turnRetry?.status, "running");
1921 + assert.strictEqual(turnRetry?.failure.message, "no first model event. Retrying in 1s.");
1922 + assert.deepStrictEqual(turnRetry?.retry, { attempt: 1, maxAttempts: 1, retryDelayMs: 1_000 });
1923 + const http = { maxAttempts: 10, errorKind: "server", httpStatus: 504, retryDelayMs: 1_000 };
1924 + const modelRetry = yield* retry("turn/modelRetryScheduled", "record-2", 1, http);
1925 + assert.strictEqual(
1926 + modelRetry?.failure.message,
1927 + "Model request failed (HTTP 504). Retrying in 1s.",
1928 + );
1929 + const nextChain = yield* retry("turn/retryScheduled", "record-3", 1, timeout);
1930 + assert.strictEqual(new Set([turnRetry?.id, modelRetry?.id, nextChain?.id]).size, 3);
1931 + const sameChain = yield* retry("turn/modelRetryScheduled", "record-4", 2, {
1932 + ...http,
1933 + httpStatus: 429,
1934 + retryDelayMs: 60_000,
1935 + });
1936 + assert.strictEqual(sameChain?.id, modelRetry?.id);
1937 + assert.deepStrictEqual(sameChain?.retry, {
1938 + attempt: 2,
1939 + maxAttempts: 9,
1940 + retryDelayMs: 60_000,
1941 + });
1942 +
1943 + yield* fake.emit("item/started", {
1944 + turnId: nativeId,
1945 + item: {
1946 + itemId: "answer",
1947 + turnId: nativeId,
1948 + kind: "agentMessage",
1949 + revision: 1,
1950 + status: "inProgress",
1951 + },
1952 + });
1953 + const recovered = [yield* takeRetryRow(), yield* takeRetryRow(), yield* takeRetryRow()];
1954 + assert.sameMembers(
1955 + recovered.map((row) => `${row?.id} ${row?.status} ${row?.failure.message}`),
1956 + [
1957 + `${turnRetry?.id} completed no first model event`,
1958 + `${nextChain?.id} completed no first model event`,
1959 + `${sameChain?.id} completed Model request failed (HTTP 429)`,
1960 + ],
1961 + );
1962 +
1963 + const stalled = yield* retry("turn/modelRetryScheduled", "record-5", 1, http);
1964 + yield* fake.emit("turn/completed", {
1965 + turnId: nativeId,
1966 + terminal: "failed",
1967 + error: { message: "Muse gave up" },
1968 + });
1969 + const stopped = yield* takeRetryRow();
1970 + assert.deepStrictEqual([stopped?.id, stopped?.status], [stalled?.id, "interrupted"]);
1971 + }).pipe(Effect.scoped, Effect.provide(testLayer)),
1972 + );
1973 +
1974 it.effect("opens a prompt-and-result child thread for a native subagent", () =>
1975 Effect.gen(function* () {
1976 const fake = yield* makeFakeMuse();
packages/provider-muse/src/server/adapter.ts
+97 -16
@@ -7,6 +7,7 @@ import {
7 type OrchestrationV2ConversationMessage,
8 type OrchestrationV2ExecutionNode,
9 type OrchestrationV2ProviderCapabilities,
10 + type OrchestrationV2ProviderRetry,
11 type OrchestrationV2ProviderSession,
12 type OrchestrationV2ProviderThread,
13 type OrchestrationV2ProviderTurn,
@@ -26,6 +27,7 @@ import {
27 import type { MuseSettings } from "../settings.ts";
28 import { AgentScope } from "@t3tools/shared/AgentScope";
29 import { getModelSelectionStringOptionValue } from "@t3tools/shared/model";
30 +import { formatDuration } from "@t3tools/shared/orchestrationTiming";
31 import * as Cause from "effect/Cause";
32 import * as DateTime from "effect/DateTime";
33 import * as Deferred from "effect/Deferred";
@@ -52,7 +54,7 @@ import {
54 MuseSubscriptionUsage,
55 MuseTokenUsageEvent,
56 MuseTodoList,
55 - MuseTurnRetryScheduled,
57 + MuseRetryScheduled,
58 MuseTurnCompleted,
59 MuseTurnStartResult,
60 MuseUserInput,
@@ -84,7 +86,10 @@ import {
86 type BackgroundWorkReport,
87 } from "@t3tools/provider-core/server/notification";
88 import type * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests";
87 -import { makeProviderFailure } from "@t3tools/provider-core/server/failure";
89 +import {
90 + makeProviderFailure,
91 + makeProviderRetryTurnItem,
92 +} from "@t3tools/provider-core/server/failure";
93 import {
94 makeSubagentChildThread,
95 makeSubagentConversationArtifacts,
@@ -214,6 +219,7 @@ interface ActiveTurn {
219 readonly started: Map<string, DateTime.Utc>;
220 readonly dirty: Set<string>;
221 readonly settledRequests: Set<string>;
222 + readonly retries: Array<MuseRetryRow>;
223 readonly done: Deferred.Deferred<void>;
224 todoPlanId?: PlanId;
225 nextOrdinal: number;
@@ -222,6 +228,15 @@ interface ActiveTurn {
228 compact: boolean;
229 }
230
231 +interface MuseRetryRow {
232 + readonly method: string;
233 + readonly nativeId: string;
234 + readonly retry: OrchestrationV2ProviderRetry;
235 + readonly cause: string;
236 + readonly code: string | undefined;
237 + running: boolean;
238 +}
239 +
240 interface PendingRequest {
241 readonly native:
242 | { type: "approval"; value: MuseApproval }
@@ -258,9 +273,12 @@ const INFORMATIONAL_NOTIFICATIONS = new Set([
273 "session/tokenUsage",
274 "session/todoListChanged",
275 "turn/retryScheduled",
276 + "turn/modelRetryScheduled",
277 "usage/changed",
278 ]);
279 const TOOLS_WITH_NATIVE_ROWS = new Set(["write_todos", "request_user_input", "workflow"]);
280 +// Reminder children and other runtime items can start while the model is still retrying.
281 +const MODEL_OUTPUT_KINDS = new Set(["agentMessage", "reasoning", "toolCall"]);
282 const responseAnswerSchema = Schema.Union([Schema.String, Schema.Array(Schema.String)]);
283
284 export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
@@ -447,6 +465,52 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
465 updatedAt: time,
466 };
467 };
468 + const emitRetry = Effect.fnUntraced(function* (
469 + turn: ActiveTurn,
470 + row: MuseRetryRow,
471 + status: "running" | "completed" | "interrupted",
472 + ) {
473 + const time = yield* DateTime.now;
474 + const base = baseItem(turn, row.nativeId, time);
475 + yield* emit({
476 + type: "turn_item.updated",
477 + driver: MUSE_PROVIDER,
478 + turnItem: {
479 + ...makeProviderRetryTurnItem({
480 + ...base,
481 + driver: MUSE_PROVIDER,
482 + nodeId: turn.input.rootNodeId,
483 + itemOrdinal: base.ordinal,
484 + failure: makeProviderFailure({
485 + // Clients show only the failure message, so a waiting row carries its delay.
486 + message:
487 + status === "running"
488 + ? `${row.cause}. Retrying in ${formatDuration(row.retry.retryDelayMs ?? 0)}.`
489 + : row.cause,
490 + code: row.code,
491 + class: "provider_error",
492 + retryable: true,
493 + }),
494 + retry: row.retry,
495 + status,
496 + updatedAt: time,
497 + }),
498 + // The shared helper keys one row per turn; a long Muse turn can retry several times.
499 + id: base.id,
500 + },
501 + });
502 + });
503 + // Muse sends no recovered event, so model output or the turn's end closes open retry rows.
504 + const settleRetries = Effect.fnUntraced(function* (
505 + turn: ActiveTurn,
506 + status: "completed" | "interrupted",
507 + ) {
508 + for (const row of turn.retries) {
509 + if (!row.running) continue;
510 + row.running = false;
511 + yield* emitRetry(turn, row, status);
512 + }
513 + });
514 const childItemIdentity = (context: ChildContext, nativeId: string) =>
515 `${options.instanceId}:child:${context.childSessionId}:${nativeId}`;
516 const itemStatus = (item: MuseItem): OrchestrationV2TurnItemStatus => {
@@ -970,6 +1034,8 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
1034 yield* publishItem(turn, item, item.status === "inProgress" ? status : undefined);
1035 }
1036 turn.dirty.clear();
1037 + // A failed turn has its own failure row; its retry rows only record that retrying stopped.
1038 + yield* settleRetries(turn, status === "completed" ? "completed" : "interrupted");
1039 for (const entry of pending.values())
1040 if (entry.request.providerTurnId !== null) yield* resolvePending(entry, "cancelled");
1041 turn.providerTurn = { ...turn.providerTurn, status, completedAt };
@@ -1413,6 +1479,8 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
1479 return;
1480 const previous = turn.items.get(item.itemId);
1481 if (previous && previous.revision >= item.revision) return;
1482 + if (method === "item/started" && MODEL_OUTPUT_KINDS.has(item.kind))
1483 + yield* settleRetries(turn, "completed");
1484 turn.items.set(item.itemId, item);
1485 turn.dirty.delete(item.itemId);
1486 yield* publishItem(turn, item);
@@ -1507,6 +1575,7 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
1575 break;
1576 case "session/tokenUsage": {
1577 const usage = yield* decode(MuseTokenUsageEvent, params);
1578 + yield* settleRetries(turn, "completed");
1579 const previous = turn.providerTurn.turnTokenUsage;
1580 const contextUsage = thread?.contextUsage;
1581 turn.providerTurn = {
@@ -1630,21 +1699,32 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
1699 });
1700 break;
1701 }
1633 - case "turn/retryScheduled": {
1634 - const retry = yield* decode(MuseTurnRetryScheduled, params);
1635 - const time = yield* DateTime.now;
1636 - yield* emit({
1637 - type: "turn_item.updated",
1638 - driver: MUSE_PROVIDER,
1639 - turnItem: {
1640 - ...baseItem(turn, `retry:${retry.nextAttempt}`, time),
1641 - type: "system_notice",
1642 - status: "completed",
1643 - title: "Muse retry",
1644 - completedAt: time,
1645 - message: `Muse is retrying this turn (attempt ${retry.nextAttempt} of ${retry.maxAttempts}): ${retry.reason}`,
1702 + case "turn/retryScheduled":
1703 + case "turn/modelRetryScheduled": {
1704 + const retry = yield* decode(MuseRetryScheduled, params);
1705 + const cause =
1706 + retry.httpStatus === undefined ? retry.errorKind : `HTTP ${retry.httpStatus}`;
1707 + // A chain's first retry opens a row; its later attempts update that row.
1708 + const index =
1709 + retry.attempt > 1 ? turn.retries.findLastIndex((row) => row.method === method) : -1;
1710 + const row: MuseRetryRow = {
1711 + method,
1712 + nativeId: turn.retries[index]?.nativeId ?? `retry:${retry.sourceRange.first.id}`,
1713 + // Muse counts the first request as attempt 1; T3 counts retries.
1714 + retry: {
1715 + attempt: retry.attempt,
1716 + maxAttempts: Math.max(1, retry.maxAttempts - 1),
1717 + retryDelayMs: Math.trunc(retry.retryDelayMs),
1718 },
1647 - });
1719 + cause:
1720 + retry.reason ??
1721 + (cause ? `Model request failed (${cause})` : "Model request failed"),
1722 + code: retry.errorKind,
1723 + running: true,
1724 + };
1725 + if (index === -1) turn.retries.push(row);
1726 + else turn.retries[index] = row;
1727 + yield* emitRetry(turn, row, "running");
1728 break;
1729 }
1730 }
@@ -2041,6 +2121,7 @@ export const makeMuseAdapterV2 = Effect.fn("makeMuseAdapterV2")(function* (
2121 started: new Map(),
2122 dirty: new Set(),
2123 settledRequests: new Set(),
2124 + retries: [],
2125 done: yield* Deferred.make<void>(),
2126 nextOrdinal: turnInput.providerTurnOrdinal * 100 + 1,
2127 flushScheduled: false,
packages/provider-muse/src/server/protocol.ts
+7 -2
@@ -182,12 +182,17 @@ export const MuseSubscriptionUsage = Schema.Struct({
182 }),
183 });
184 export type MuseSubscriptionUsage = typeof MuseSubscriptionUsage.Type;
185 -export const MuseTurnRetryScheduled = Schema.Struct({
185 +/** `turn/retryScheduled` carries `reason`; `turn/modelRetryScheduled` carries the HTTP failure. */
186 +export const MuseRetryScheduled = Schema.Struct({
187 turnId: NonEmptyString,
188 + attempt: Schema.Int.check(Schema.isGreaterThan(0)),
189 nextAttempt: Schema.Int.check(Schema.isGreaterThan(0)),
190 maxAttempts: Schema.Int.check(Schema.isGreaterThan(0)),
189 - reason: NonEmptyString,
191 + reason: Schema.optional(NonEmptyString),
192 + errorKind: Schema.optional(Schema.String),
193 + httpStatus: Schema.optional(Schema.Int),
194 retryDelayMs: Schema.Finite.check(Schema.isGreaterThanOrEqualTo(0)),
195 + sourceRange: Schema.Struct({ first: Schema.Struct({ id: NonEmptyString }) }),
196 });
197 export const MuseCompactResult = Schema.Struct({
198 status: Schema.String,