@devries-cameron20 / t3code-1 / commits / 00766e2c02
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): fail any provider turn stuck in a retry (#33152)

maria committed Nov 11, 2026 at 21:19 UTC 00766e2c027ac095e604da81b958113287f2f6b1
6 files changed +426 -10
apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts
+14 -3
@@ -10256,11 +10256,15 @@ describe("ClaudeAdapterV2 background wake turns", () => {
10256 retryDelayMs: 1_500,
10257 });
10258
10259 + // The stream starting again settles the retry before any assistant message completes.
10260 yield* Queue.offer(
10261 harness.sdkMessages,
10261 - makeAssistantTextFrame({
10262 - uuid: "00000000-0000-4000-8000-000000000202",
10263 - text: "Opening GitHub.",
10262 + claudeSdkFrame({
10263 + type: "stream_event",
10264 + event: { type: "message_start", message: { id: "api-retry-recovered-message" } },
10265 + parent_tool_use_id: null,
10266 + session_id: WAKE_NATIVE_SESSION,
10267 + uuid: "00000000-0000-4000-8000-000000000204",
10268 }),
10269 );
10270 yield* harness.until(() => retryItems().length === 2);
@@ -10269,6 +10273,13 @@ describe("ClaudeAdapterV2 background wake turns", () => {
10273 assert.equal(recoveredRetry?.id, runningRetry?.id);
10274 assert.equal(recoveredRetry?.status, "completed");
10275 assert.equal(recoveredRetry?.title, "Provider recovered");
10276 + yield* Queue.offer(
10277 + harness.sdkMessages,
10278 + makeAssistantTextFrame({
10279 + uuid: "00000000-0000-4000-8000-000000000202",
10280 + text: "Opening GitHub.",
10281 + }),
10282 + );
10283
10284 yield* Queue.offer(
10285 harness.sdkMessages,
apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts
+10
@@ -10430,6 +10430,16 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
10430 const message = input.message;
10431 const context = yield* Ref.get(activeTurn);
10432 const liveQuery = yield* Ref.get(queryContext);
10433 + // The API answering again is recovery; a long text or tool input block emits nothing
10434 + // else. Judged here, in arrival order: held and buffered frames are handled later.
10435 + if (
10436 + context !== null &&
10437 + liveQuery?.query === input.query &&
10438 + message.type === "stream_event" &&
10439 + message.event.type === "message_start"
10440 + ) {
10441 + yield* completeProviderRetry(context, yield* DateTime.now);
10442 + }
10443 if (
10444 context !== null &&
10445 context.pendingSteers.size > 0 &&
apps/server/src/orchestration-v2/ProviderSessionManager.test.ts
+193
@@ -55,6 +55,10 @@ import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
55 import * as ProviderSessionManager from "./ProviderSessionManager.ts";
56 import * as CommandOutputHub from "./CommandOutputHub.ts";
57 import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
58 +import {
59 + makeProviderFailure,
60 + makeProviderRetryTurnItem,
61 +} from "@t3tools/provider-core/server/failure";
62
63 const layerTestDatabase = SqlitePersistence.layerMemory;
64 const layerTestStores = Layer.merge(EventStore.layer, ProjectionStore.layer).pipe(
@@ -2023,6 +2027,195 @@ it.effect("ProviderSessionManagerV2 drains subscribers when the provider stops",
2027 }),
2028 );
2029
2030 +it.effect("ProviderSessionManagerV2 fails a turn whose provider retry never resumes", () =>
2031 + Effect.gen(function* () {
2032 + const state = yield* Ref.make(emptyState);
2033 + const interrupted = yield* Deferred.make<void>();
2034 + const effect = Effect.gen(function* () {
2035 + const eventSink = yield* EventSink.EventSinkV2;
2036 + const idAllocator = yield* IdAllocator.IdAllocatorV2;
2037 + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
2038 + const now = yield* DateTime.now;
2039 + const threadId = ThreadId.make("thread-provider-session-manager-retry-stall");
2040 + const providerSessionId = yield* idAllocator.allocate.providerSession({
2041 + providerInstanceId: modelSelection.instanceId,
2042 + threadId,
2043 + });
2044 + const providerThread = makeProviderThread({ idAllocator, threadId, providerSessionId, now });
2045 + yield* eventSink.write({
2046 + events: [
2047 + yield* makeThreadCreatedEvent({ idAllocator, threadId, now }),
2048 + {
2049 + id: yield* idAllocator.allocate.event({ threadId }),
2050 + type: "provider-thread.updated",
2051 + threadId,
2052 + occurredAt: now,
2053 + payload: providerThread,
2054 + },
2055 + ],
2056 + });
2057 + const runtime = yield* manager.open({
2058 + threadId,
2059 + providerSessionId,
2060 + modelSelection,
2061 + runtimePolicy,
2062 + });
2063 + const subscription = yield* runtime.subscribeEvents!;
2064 + const published = yield* Queue.unbounded<void>();
2065 + // The pump updates the stall watch before it publishes a turn item.
2066 + const collected = yield* subscription.events.pipe(
2067 + Stream.tap((event) =>
2068 + event.type === "turn_item.updated" ? Queue.offer(published, undefined) : Effect.void,
2069 + ),
2070 + Stream.runCollect,
2071 + Effect.forkScoped,
2072 + );
2073 + const adapterQueue = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId));
2074 + assert.isDefined(adapterQueue);
2075 + const providerTurnId = idAllocator.derive.providerTurn({
2076 + driver: CODEX_DRIVER,
2077 + nativeTurnId: "retry-stall-turn",
2078 + });
2079 + const failure = makeProviderFailure({
2080 + message: "Rate limit exceeded",
2081 + code: "rate_limited",
2082 + class: "usage_limit",
2083 + retryable: true,
2084 + });
2085 + const retry = { attempt: 2, maxAttempts: 5, retryDelayMs: 60_000 };
2086 + const retryItem = (status: "running" | "interrupted") =>
2087 + makeProviderRetryTurnItem({
2088 + driver: CODEX_DRIVER,
2089 + threadId,
2090 + runId: null,
2091 + nodeId: null,
2092 + providerThreadId: providerThread.id,
2093 + providerTurnId,
2094 + itemOrdinal: 3,
2095 + failure,
2096 + retry,
2097 + status,
2098 + startedAt: now,
2099 + updatedAt: now,
2100 + });
2101 + yield* Queue.offer(adapterQueue!, {
2102 + type: "turn_item.updated",
2103 + driver: CODEX_DRIVER,
2104 + turnItem: retryItem("running"),
2105 + });
2106 + yield* Queue.take(published);
2107 + const offerOutput = (outputThreadId: ThreadId) =>
2108 + Queue.offer(adapterQueue!, {
2109 + type: "turn_item.updated",
2110 + driver: CODEX_DRIVER,
2111 + turnItem: {
2112 + id: idAllocator.derive.turnItemFromProviderItem({
2113 + driver: CODEX_DRIVER,
2114 + nativeItemId: `retry-stall-output-${outputThreadId}`,
2115 + }),
2116 + threadId: outputThreadId,
2117 + runId: null,
2118 + nodeId: null,
2119 + providerThreadId: providerThread.id,
2120 + providerTurnId,
2121 + nativeItemRef: null,
2122 + parentItemId: null,
2123 + ordinal: 4,
2124 + type: "compaction",
2125 + driver: CODEX_DRIVER,
2126 + status: "running",
2127 + title: "Compacting context",
2128 + startedAt: now,
2129 + completedAt: null,
2130 + updatedAt: now,
2131 + },
2132 + }).pipe(Effect.andThen(Queue.take(published)));
2133 + // The stall window is 15 minutes plus the provider's announced retry delay.
2134 + yield* TestClock.adjust("15 minutes");
2135 + assert.equal((yield* Ref.get(state)).interruptCount, 0);
2136 + // Output on the retry's thread proves the provider is alive, so a new window starts.
2137 + yield* offerOutput(threadId);
2138 + yield* TestClock.adjust("1 minute");
2139 + // Another thread's output lets the timer start its next wait and does not count.
2140 + yield* offerOutput(ThreadId.make("thread-provider-session-manager-retry-stall-other"));
2141 + yield* TestClock.adjust("15 minutes");
2142 + assert.equal((yield* Ref.get(state)).interruptCount, 0);
2143 + yield* TestClock.adjust("1 minute");
2144 + yield* Deferred.await(interrupted);
2145 + yield* Queue.offer(adapterQueue!, {
2146 + type: "turn_item.updated",
2147 + driver: CODEX_DRIVER,
2148 + turnItem: retryItem("interrupted"),
2149 + });
2150 + yield* Queue.offer(adapterQueue!, {
2151 + type: "provider_turn.updated",
2152 + driver: CODEX_DRIVER,
2153 + threadId,
2154 + providerTurn: {
2155 + id: providerTurnId,
2156 + providerThreadId: providerThread.id,
2157 + nodeId: idAllocator.derive.rootNode({
2158 + runId: idAllocator.derive.run({ threadId, ordinal: 1 }),
2159 + }),
2160 + runAttemptId: null,
2161 + nativeTurnRef: null,
2162 + ordinal: 1,
2163 + status: "interrupted",
2164 + startedAt: now,
2165 + completedAt: now,
2166 + },
2167 + });
2168 + yield* Queue.offer(adapterQueue!, {
2169 + type: "turn.terminal",
2170 + driver: CODEX_DRIVER,
2171 + providerThreadId: providerThread.id,
2172 + providerTurnId,
2173 + runOrdinal: 1,
2174 + status: "interrupted",
2175 + failure: null,
2176 + threadDisposition: "reusable",
2177 + });
2178 + yield* Queue.offer(adapterQueue!, {
2179 + type: "provider_session.updated",
2180 + driver: CODEX_DRIVER,
2181 + providerSession: { ...runtime.providerSession, status: "stopped", updatedAt: now },
2182 + });
2183 + yield* Queue.end(adapterQueue!);
2184 +
2185 + const events = Array.from(yield* Fiber.join(collected));
2186 + const terminal = events.find((event) => event.type === "turn.terminal");
2187 + const providerTurn = events.find((event) => event.type === "provider_turn.updated");
2188 + assert.equal(
2189 + providerTurn?.type === "provider_turn.updated" ? providerTurn.providerTurn.status : null,
2190 + "failed",
2191 + );
2192 + assert.deepEqual(terminal, {
2193 + type: "turn.terminal",
2194 + driver: CODEX_DRIVER,
2195 + providerThreadId: providerThread.id,
2196 + providerTurnId,
2197 + runOrdinal: 1,
2198 + failureItemOrdinal: 3,
2199 + status: "failed",
2200 + failure: { ...failure, retryable: false },
2201 + retry,
2202 + retryStartedAt: now,
2203 + threadDisposition: "reusable",
2204 + });
2205 + });
2206 +
2207 + yield* effect.pipe(
2208 + Effect.provide(
2209 + layerTest({
2210 + state,
2211 + idleTimeoutMs: 60 * 60_000,
2212 + interruptTurn: Deferred.succeed(interrupted, undefined).pipe(Effect.asVoid),
2213 + }),
2214 + ),
2215 + );
2216 + }),
2217 +);
2218 +
2219 it.effect(
2220 "ProviderSessionManagerV2 issues MCP credentials before opening and revokes them on close",
2221 () =>
apps/server/src/orchestration-v2/ProviderSessionManager.ts
+197 -6
@@ -12,7 +12,9 @@ import {
12 ProviderInstanceId,
13 ProviderSessionId,
14 ThreadId,
15 + type OrchestrationV2TurnItem,
16 type ProviderThreadId,
17 + type ProviderTurnId,
18 } from "@t3tools/contracts";
19 import * as Cause from "effect/Cause";
20 import * as Clock from "effect/Clock";
@@ -66,6 +68,14 @@ const busyTurnPrefix = (providerThreadId: ProviderThreadId) => `${providerThread
68 const busyTurnKey = (providerThreadId: ProviderThreadId, runOrdinal: number) =>
69 `${busyTurnPrefix(providerThreadId)}${runOrdinal}`;
70 const UNLOAD_THREAD_TIMEOUT_MS = 10 * 1000;
71 +// Providers can announce a retry and never resume. Longer than any adapter's own stall handling.
72 +const PROVIDER_RETRY_STALL_TIMEOUT = Duration.minutes(15);
73 +type ProviderErrorItem = Extract<OrchestrationV2TurnItem, { readonly type: "error" }>;
74 +type ProviderRetryItem = ProviderErrorItem & {
75 + readonly providerTurnId: ProviderTurnId;
76 + readonly providerThreadId: ProviderThreadId;
77 + readonly retry: NonNullable<ProviderErrorItem["retry"]>;
78 +};
79 /** Suggestions outlive their sessions, so keep only the most recently updated threads. */
80 const MAX_PROMPT_SUGGESTIONS = 256;
81 const WORKING_RUN_STATUSES: ReadonlySet<string> = new Set([
@@ -2203,21 +2213,198 @@ export const layerWithOptions = (
2213 ),
2214 );
2215
2216 + const makeRetryStallWatch = (entry: LiveSessionEntry) => {
2217 + const runningRetries = new Map<ProviderTurnId, ProviderRetryItem>();
2218 + const stalledRetries = new Map<ProviderTurnId, ProviderRetryItem>();
2219 + const stallTimers = new Map<ProviderTurnId, Fiber.Fiber<void>>();
2220 + // Claude and Pi keep the retry row running while a recovered reply streams, so any
2221 + // event on the retry's thread counts as the provider being alive.
2222 + const activeSinceCheck = new Set<ProviderTurnId>();
2223 + const noteThreadActivity = (event: ProviderAdapter.ProviderAdapterV2Event) => {
2224 + if (runningRetries.size === 0) return;
2225 + const threadId =
2226 + event.type === "turn_item.updated"
2227 + ? event.turnItem.threadId
2228 + : event.type === "message.updated"
2229 + ? event.message.threadId
2230 + : event.type === "node.updated"
2231 + ? event.node.threadId
2232 + : event.type === "plan.updated"
2233 + ? event.plan.threadId
2234 + : undefined;
2235 + if (threadId === undefined) return;
2236 + for (const item of runningRetries.values()) {
2237 + if (item.threadId === threadId) activeSinceCheck.add(item.providerTurnId);
2238 + }
2239 + };
2240 + // Stops the stalled turn; its interrupted terminal is then reported as the retry's failure.
2241 + const stopStalledRetry = Effect.fn("ProviderSessionManager.stopStalledRetry")(
2242 + function* (item: ProviderRetryItem) {
2243 + if (runningRetries.get(item.providerTurnId) !== item) return;
2244 + // The terminal this stop causes must not interrupt the stop itself.
2245 + stallTimers.delete(item.providerTurnId);
2246 + const current = (yield* Ref.get(sessions)).get(
2247 + sessionKey(entry.runtime.providerSessionId),
2248 + );
2249 + if (current?.runtime !== entry.runtime) return;
2250 + const { providerThread } = yield* projectionStore.getProviderControlContext(
2251 + item.threadId,
2252 + { providerThreadId: item.providerThreadId, providerTurnId: item.providerTurnId },
2253 + );
2254 + if (providerThread === undefined || runningRetries.get(item.providerTurnId) !== item)
2255 + return;
2256 + stalledRetries.set(item.providerTurnId, item);
2257 + yield* entry.exposedRuntime
2258 + .interruptTurn({ providerThread, providerTurnId: item.providerTurnId })
2259 + .pipe(
2260 + // A slow stop can still end the turn later, so that terminal keeps the retry failure.
2261 + Effect.catchTags({
2262 + ProviderAdapterInterruptTimeoutError: (cause) =>
2263 + Effect.logWarning("orchestration-v2.provider-retry-stall-stop-timed-out", {
2264 + providerSessionId: entry.runtime.providerSessionId,
2265 + cause,
2266 + }),
2267 + }),
2268 + );
2269 + },
2270 + (effect, item) =>
2271 + effect.pipe(
2272 + Effect.catchCause((cause) =>
2273 + Effect.sync(() => stalledRetries.delete(item.providerTurnId)).pipe(
2274 + Effect.andThen(
2275 + Effect.logWarning("orchestration-v2.provider-retry-stall-stop-failed", {
2276 + providerSessionId: entry.runtime.providerSessionId,
2277 + cause,
2278 + }),
2279 + ),
2280 + ),
2281 + ),
2282 + ),
2283 + );
2284 + const stopStallTimer = (providerTurnId: ProviderTurnId) => {
2285 + const fiber = stallTimers.get(providerTurnId);
2286 + stallTimers.delete(providerTurnId);
2287 + activeSinceCheck.delete(providerTurnId);
2288 + return fiber === undefined ? Effect.void : Fiber.interrupt(fiber);
2289 + };
2290 + const trackProviderRetry = (
2291 + event: ProviderAdapter.ProviderAdapterV2Event,
2292 + ): {
2293 + readonly event: ProviderAdapter.ProviderAdapterV2Event;
2294 + readonly armed?: ProviderRetryItem;
2295 + readonly settled?: ProviderTurnId;
2296 + } => {
2297 + noteThreadActivity(event);
2298 + if (event.type === "turn_item.updated") {
2299 + const item = event.turnItem;
2300 + if (
2301 + item.type !== "error" ||
2302 + item.retry === undefined ||
2303 + item.providerTurnId === null ||
2304 + item.providerThreadId === null
2305 + )
2306 + return { event };
2307 + const retryItem = {
2308 + ...item,
2309 + providerTurnId: item.providerTurnId,
2310 + providerThreadId: item.providerThreadId,
2311 + retry: item.retry,
2312 + };
2313 + if (item.status === "running") {
2314 + runningRetries.set(retryItem.providerTurnId, retryItem);
2315 + return { event, armed: retryItem };
2316 + }
2317 + // A stall marker is set only once the stop is sent, so only the terminal clears it.
2318 + runningRetries.delete(retryItem.providerTurnId);
2319 + return { event, settled: retryItem.providerTurnId };
2320 + }
2321 + if (event.type === "provider_turn.updated") {
2322 + if (event.providerTurn.status === "running") return { event };
2323 + runningRetries.delete(event.providerTurn.id);
2324 + const settled = event.providerTurn.id;
2325 + return stalledRetries.has(settled) &&
2326 + (event.providerTurn.status === "interrupted" ||
2327 + event.providerTurn.status === "cancelled")
2328 + ? {
2329 + event: { ...event, providerTurn: { ...event.providerTurn, status: "failed" } },
2330 + settled,
2331 + }
2332 + : { event, settled };
2333 + }
2334 + if (event.type !== "turn.terminal") return { event };
2335 + runningRetries.delete(event.providerTurnId);
2336 + const stalled = stalledRetries.get(event.providerTurnId);
2337 + stalledRetries.delete(event.providerTurnId);
2338 + const settled = event.providerTurnId;
2339 + if (
2340 + stalled === undefined ||
2341 + (event.status !== "interrupted" && event.status !== "cancelled")
2342 + )
2343 + return { event, settled };
2344 + return {
2345 + settled,
2346 + event: {
2347 + type: "turn.terminal",
2348 + driver: event.driver,
2349 + providerThreadId: event.providerThreadId,
2350 + providerTurnId: event.providerTurnId,
2351 + runOrdinal: event.runOrdinal,
2352 + failureItemOrdinal: stalled.ordinal,
2353 + status: "failed",
2354 + failure: { ...stalled.failure, retryable: false },
2355 + retry: stalled.retry,
2356 + ...(stalled.startedAt === null ? {} : { retryStartedAt: stalled.startedAt }),
2357 + threadDisposition: event.threadDisposition,
2358 + },
2359 + };
2360 + };
2361 + return {
2362 + track: trackProviderRetry,
2363 + // Runs for every provider event; spans would be per-event noise. Timers are children
2364 + // of the pump fiber, so a released session takes them down.
2365 + update: Effect.fnUntraced(function* (tracked: ReturnType<typeof trackProviderRetry>) {
2366 + const item = tracked.armed;
2367 + if (item === undefined) {
2368 + if (tracked.settled !== undefined) yield* stopStallTimer(tracked.settled);
2369 + return;
2370 + }
2371 + yield* stopStallTimer(item.providerTurnId);
2372 + const window = Duration.sum(
2373 + PROVIDER_RETRY_STALL_TIMEOUT,
2374 + Duration.millis(item.retry.retryDelayMs ?? 0),
2375 + );
2376 + const timer = yield* Effect.gen(function* () {
2377 + // Only a thread that stayed silent for a whole window has stalled.
2378 + do {
2379 + activeSinceCheck.delete(item.providerTurnId);
2380 + yield* Effect.sleep(window);
2381 + } while (activeSinceCheck.has(item.providerTurnId));
2382 + yield* stopStalledRetry(item);
2383 + }).pipe(Effect.forkChild({ startImmediately: true }));
2384 + stallTimers.set(item.providerTurnId, timer);
2385 + }),
2386 + };
2387 + };
2388 +
2389 const startEventPump = (entry: LiveSessionEntry) => {
2390 let stoppedByProvider = false;
2391 let lastSessionStatus = entry.runtime.providerSession.status;
2392 + const retryStalls = makeRetryStallWatch(entry);
2393 return entry.runtime.events.pipe(
2210 - Stream.runForEach((event) => {
2394 + Stream.runForEach((nativeEvent) => {
2395 if (shutdownSignal.received) return Effect.void;
2396 // Live output bypasses persistence and subscribers so floods cannot delay domain events.
2213 - if (event.type === "command_output.delta") {
2214 - return commandOutput.append(event);
2397 + if (nativeEvent.type === "command_output.delta") {
2398 + return commandOutput.append(nativeEvent);
2399 }
2216 - if (event.type === "prompt_suggestion.updated") {
2400 + if (nativeEvent.type === "prompt_suggestion.updated") {
2401 return SubscriptionRef.update(promptSuggestionStore, (current) => {
2402 const next = new Map(current);
2219 - next.delete(event.threadId);
2220 - next.set(event.threadId, { runId: event.runId, text: event.text });
2403 + next.delete(nativeEvent.threadId);
2404 + next.set(nativeEvent.threadId, {
2405 + runId: nativeEvent.runId,
2406 + text: nativeEvent.text,
2407 + });
2408 for (const threadId of next.keys()) {
2409 if (next.size <= MAX_PROMPT_SUGGESTIONS) break;
2410 next.delete(threadId);
@@ -2225,6 +2412,9 @@ export const layerWithOptions = (
2412 return next;
2413 });
2414 }
2415 + const tracked = retryStalls.track(nativeEvent);
2416 + const event = tracked.event;
2417 + const armRetryStall = retryStalls.update(tracked);
2418 const releaseLiveOutput =
2419 event.type === "turn_item.updated" &&
2420 event.turnItem.type === "command_execution" &&
@@ -2254,6 +2444,7 @@ export const layerWithOptions = (
2444 ? Effect.void
2445 : touchActivity(entry.runtime.providerSessionId, { providerActivity: true }),
2446 ).pipe(
2447 + Effect.andThen(armRetryStall),
2448 Effect.andThen(
2449 event.type === "provider_session.updated"
2450 ? persistProviderSessionUpdate(entry, event)
packages/provider-pi/src/server/adapter.test.ts
+6 -1
@@ -4744,7 +4744,11 @@ describe("PiAdapterV2", () => {
4744 const running = yield* takeEvent(
4745 (event) => event.type === "turn_item.updated" && event.turnItem.type === "error",
4746 );
4747 - yield* fake.emit({ type: "auto_retry_end", success: true, attempt: 1 });
4747 + // Streamed tool input settles the retry; Pi sends auto_retry_end only after the message ends.
4748 + yield* fake.emit({
4749 + type: "message_update",
4750 + assistantMessageEvent: { type: "toolcall_delta", contentIndex: 0, delta: "{" },
4751 + });
4752 const recovered = yield* takeEvent(
4753 (event) => event.type === "turn_item.updated" && event.turnItem.type === "error",
4754 );
@@ -4757,6 +4761,7 @@ describe("PiAdapterV2", () => {
4761 recovered.turnItem.status === "completed" &&
4762 recovered.turnItem.title === "Provider recovered",
4763 );
4764 + yield* fake.emit({ type: "auto_retry_end", success: true, attempt: 1 });
4765
4766 yield* fake.emit({ type: "agent_settled" });
4767 const terminal = yield* takeEvent((event) => event.type === "turn.terminal");
packages/provider-pi/src/server/adapter.ts
+6
@@ -1967,6 +1967,12 @@ export const makePiAdapterV2 = Effect.fn("makePiAdapterV2")(function* (
1967 case "message_update": {
1968 if (turn === null) return;
1969 turn.sawAgentActivity = true;
1970 + // Pi sends auto_retry_end only after the message ends; streamed output is recovery.
1971 + if (turn.activeProviderRetry !== null) {
1972 + const recoveredAt = yield* DateTime.now;
1973 + yield* emitProviderRetry(turn, turn.activeProviderRetry, "completed", recoveredAt);
1974 + turn.activeProviderRetry = null;
1975 + }
1976 yield* reportLiveUsage(turn, event["usage"]);
1977 const delta = event["assistantMessageEvent"];
1978 const deltaType = recordString(delta, "type");