The best way to control Claude Code, Codex and more from any computer or phone
fix(server): fail any provider turn stuck in a retry (#33152)
maria committed
Nov 11, 2026 at 21:19 UTC
00766e2c027ac095e604da81b958113287f2f6b1
6 files changed
+426
-10
apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts
+14
-3
@@ -10256,11 +10256,15 @@ describe("ClaudeAdapterV2 background wake turns", () => {
10256
retryDelayMs: 1_500,
10257
});
10258
10259
+ // The stream starting again settles the retry before any assistant message completes.
10260
yield* Queue.offer(
10261
harness.sdkMessages,
10261
- makeAssistantTextFrame({
10262
- uuid: "00000000-0000-4000-8000-000000000202",
10263
- text: "Opening GitHub.",
10262
+ claudeSdkFrame({
10263
+ type: "stream_event",
10264
+ event: { type: "message_start", message: { id: "api-retry-recovered-message" } },
10265
+ parent_tool_use_id: null,
10266
+ session_id: WAKE_NATIVE_SESSION,
10267
+ uuid: "00000000-0000-4000-8000-000000000204",
10268
}),
10269
);
10270
yield* harness.until(() => retryItems().length === 2);
@@ -10269,6 +10273,13 @@ describe("ClaudeAdapterV2 background wake turns", () => {
10273
assert.equal(recoveredRetry?.id, runningRetry?.id);
10274
assert.equal(recoveredRetry?.status, "completed");
10275
assert.equal(recoveredRetry?.title, "Provider recovered");
10276
+ yield* Queue.offer(
10277
+ harness.sdkMessages,
10278
+ makeAssistantTextFrame({
10279
+ uuid: "00000000-0000-4000-8000-000000000202",
10280
+ text: "Opening GitHub.",
10281
+ }),
10282
+ );
10283
10284
yield* Queue.offer(
10285
harness.sdkMessages,
apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts
+10
@@ -10430,6 +10430,16 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
10430
const message = input.message;
10431
const context = yield* Ref.get(activeTurn);
10432
const liveQuery = yield* Ref.get(queryContext);
10433
+ // The API answering again is recovery; a long text or tool input block emits nothing
10434
+ // else. Judged here, in arrival order: held and buffered frames are handled later.
10435
+ if (
10436
+ context !== null &&
10437
+ liveQuery?.query === input.query &&
10438
+ message.type === "stream_event" &&
10439
+ message.event.type === "message_start"
10440
+ ) {
10441
+ yield* completeProviderRetry(context, yield* DateTime.now);
10442
+ }
10443
if (
10444
context !== null &&
10445
context.pendingSteers.size > 0 &&
apps/server/src/orchestration-v2/ProviderSessionManager.test.ts
+193
@@ -55,6 +55,10 @@ import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
55
import * as ProviderSessionManager from "./ProviderSessionManager.ts";
56
import * as CommandOutputHub from "./CommandOutputHub.ts";
57
import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
58
+import {
59
+ makeProviderFailure,
60
+ makeProviderRetryTurnItem,
61
+} from "@t3tools/provider-core/server/failure";
62
63
const layerTestDatabase = SqlitePersistence.layerMemory;
64
const layerTestStores = Layer.merge(EventStore.layer, ProjectionStore.layer).pipe(
@@ -2023,6 +2027,195 @@ it.effect("ProviderSessionManagerV2 drains subscribers when the provider stops",
2027
}),
2028
);
2029
2030
+it.effect("ProviderSessionManagerV2 fails a turn whose provider retry never resumes", () =>
2031
+ Effect.gen(function* () {
2032
+ const state = yield* Ref.make(emptyState);
2033
+ const interrupted = yield* Deferred.make<void>();
2034
+ const effect = Effect.gen(function* () {
2035
+ const eventSink = yield* EventSink.EventSinkV2;
2036
+ const idAllocator = yield* IdAllocator.IdAllocatorV2;
2037
+ const manager = yield* ProviderSessionManager.ProviderSessionManagerV2;
2038
+ const now = yield* DateTime.now;
2039
+ const threadId = ThreadId.make("thread-provider-session-manager-retry-stall");
2040
+ const providerSessionId = yield* idAllocator.allocate.providerSession({
2041
+ providerInstanceId: modelSelection.instanceId,
2042
+ threadId,
2043
+ });
2044
+ const providerThread = makeProviderThread({ idAllocator, threadId, providerSessionId, now });
2045
+ yield* eventSink.write({
2046
+ events: [
2047
+ yield* makeThreadCreatedEvent({ idAllocator, threadId, now }),
2048
+ {
2049
+ id: yield* idAllocator.allocate.event({ threadId }),
2050
+ type: "provider-thread.updated",
2051
+ threadId,
2052
+ occurredAt: now,
2053
+ payload: providerThread,
2054
+ },
2055
+ ],
2056
+ });
2057
+ const runtime = yield* manager.open({
2058
+ threadId,
2059
+ providerSessionId,
2060
+ modelSelection,
2061
+ runtimePolicy,
2062
+ });
2063
+ const subscription = yield* runtime.subscribeEvents!;
2064
+ const published = yield* Queue.unbounded<void>();
2065
+ // The pump updates the stall watch before it publishes a turn item.
2066
+ const collected = yield* subscription.events.pipe(
2067
+ Stream.tap((event) =>
2068
+ event.type === "turn_item.updated" ? Queue.offer(published, undefined) : Effect.void,
2069
+ ),
2070
+ Stream.runCollect,
2071
+ Effect.forkScoped,
2072
+ );
2073
+ const adapterQueue = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId));
2074
+ assert.isDefined(adapterQueue);
2075
+ const providerTurnId = idAllocator.derive.providerTurn({
2076
+ driver: CODEX_DRIVER,
2077
+ nativeTurnId: "retry-stall-turn",
2078
+ });
2079
+ const failure = makeProviderFailure({
2080
+ message: "Rate limit exceeded",
2081
+ code: "rate_limited",
2082
+ class: "usage_limit",
2083
+ retryable: true,
2084
+ });
2085
+ const retry = { attempt: 2, maxAttempts: 5, retryDelayMs: 60_000 };
2086
+ const retryItem = (status: "running" | "interrupted") =>
2087
+ makeProviderRetryTurnItem({
2088
+ driver: CODEX_DRIVER,
2089
+ threadId,
2090
+ runId: null,
2091
+ nodeId: null,
2092
+ providerThreadId: providerThread.id,
2093
+ providerTurnId,
2094
+ itemOrdinal: 3,
2095
+ failure,
2096
+ retry,
2097
+ status,
2098
+ startedAt: now,
2099
+ updatedAt: now,
2100
+ });
2101
+ yield* Queue.offer(adapterQueue!, {
2102
+ type: "turn_item.updated",
2103
+ driver: CODEX_DRIVER,
2104
+ turnItem: retryItem("running"),
2105
+ });
2106
+ yield* Queue.take(published);
2107
+ const offerOutput = (outputThreadId: ThreadId) =>
2108
+ Queue.offer(adapterQueue!, {
2109
+ type: "turn_item.updated",
2110
+ driver: CODEX_DRIVER,
2111
+ turnItem: {
2112
+ id: idAllocator.derive.turnItemFromProviderItem({
2113
+ driver: CODEX_DRIVER,
2114
+ nativeItemId: `retry-stall-output-${outputThreadId}`,
2115
+ }),
2116
+ threadId: outputThreadId,
2117
+ runId: null,
2118
+ nodeId: null,
2119
+ providerThreadId: providerThread.id,
2120
+ providerTurnId,
2121
+ nativeItemRef: null,
2122
+ parentItemId: null,
2123
+ ordinal: 4,
2124
+ type: "compaction",
2125
+ driver: CODEX_DRIVER,
2126
+ status: "running",
2127
+ title: "Compacting context",
2128
+ startedAt: now,
2129
+ completedAt: null,
2130
+ updatedAt: now,
2131
+ },
2132
+ }).pipe(Effect.andThen(Queue.take(published)));
2133
+ // The stall window is 15 minutes plus the provider's announced retry delay.
2134
+ yield* TestClock.adjust("15 minutes");
2135
+ assert.equal((yield* Ref.get(state)).interruptCount, 0);
2136
+ // Output on the retry's thread proves the provider is alive, so a new window starts.
2137
+ yield* offerOutput(threadId);
2138
+ yield* TestClock.adjust("1 minute");
2139
+ // Another thread's output lets the timer start its next wait and does not count.
2140
+ yield* offerOutput(ThreadId.make("thread-provider-session-manager-retry-stall-other"));
2141
+ yield* TestClock.adjust("15 minutes");
2142
+ assert.equal((yield* Ref.get(state)).interruptCount, 0);
2143
+ yield* TestClock.adjust("1 minute");
2144
+ yield* Deferred.await(interrupted);
2145
+ yield* Queue.offer(adapterQueue!, {
2146
+ type: "turn_item.updated",
2147
+ driver: CODEX_DRIVER,
2148
+ turnItem: retryItem("interrupted"),
2149
+ });
2150
+ yield* Queue.offer(adapterQueue!, {
2151
+ type: "provider_turn.updated",
2152
+ driver: CODEX_DRIVER,
2153
+ threadId,
2154
+ providerTurn: {
2155
+ id: providerTurnId,
2156
+ providerThreadId: providerThread.id,
2157
+ nodeId: idAllocator.derive.rootNode({
2158
+ runId: idAllocator.derive.run({ threadId, ordinal: 1 }),
2159
+ }),
2160
+ runAttemptId: null,
2161
+ nativeTurnRef: null,
2162
+ ordinal: 1,
2163
+ status: "interrupted",
2164
+ startedAt: now,
2165
+ completedAt: now,
2166
+ },
2167
+ });
2168
+ yield* Queue.offer(adapterQueue!, {
2169
+ type: "turn.terminal",
2170
+ driver: CODEX_DRIVER,
2171
+ providerThreadId: providerThread.id,
2172
+ providerTurnId,
2173
+ runOrdinal: 1,
2174
+ status: "interrupted",
2175
+ failure: null,
2176
+ threadDisposition: "reusable",
2177
+ });
2178
+ yield* Queue.offer(adapterQueue!, {
2179
+ type: "provider_session.updated",
2180
+ driver: CODEX_DRIVER,
2181
+ providerSession: { ...runtime.providerSession, status: "stopped", updatedAt: now },
2182
+ });
2183
+ yield* Queue.end(adapterQueue!);
2184
+
2185
+ const events = Array.from(yield* Fiber.join(collected));
2186
+ const terminal = events.find((event) => event.type === "turn.terminal");
2187
+ const providerTurn = events.find((event) => event.type === "provider_turn.updated");
2188
+ assert.equal(
2189
+ providerTurn?.type === "provider_turn.updated" ? providerTurn.providerTurn.status : null,
2190
+ "failed",
2191
+ );
2192
+ assert.deepEqual(terminal, {
2193
+ type: "turn.terminal",
2194
+ driver: CODEX_DRIVER,
2195
+ providerThreadId: providerThread.id,
2196
+ providerTurnId,
2197
+ runOrdinal: 1,
2198
+ failureItemOrdinal: 3,
2199
+ status: "failed",
2200
+ failure: { ...failure, retryable: false },
2201
+ retry,
2202
+ retryStartedAt: now,
2203
+ threadDisposition: "reusable",
2204
+ });
2205
+ });
2206
+
2207
+ yield* effect.pipe(
2208
+ Effect.provide(
2209
+ layerTest({
2210
+ state,
2211
+ idleTimeoutMs: 60 * 60_000,
2212
+ interruptTurn: Deferred.succeed(interrupted, undefined).pipe(Effect.asVoid),
2213
+ }),
2214
+ ),
2215
+ );
2216
+ }),
2217
+);
2218
+
2219
it.effect(
2220
"ProviderSessionManagerV2 issues MCP credentials before opening and revokes them on close",
2221
() =>
apps/server/src/orchestration-v2/ProviderSessionManager.ts
+197
-6
@@ -12,7 +12,9 @@ import {
12
ProviderInstanceId,
13
ProviderSessionId,
14
ThreadId,
15
+ type OrchestrationV2TurnItem,
16
type ProviderThreadId,
17
+ type ProviderTurnId,
18
} from "@t3tools/contracts";
19
import * as Cause from "effect/Cause";
20
import * as Clock from "effect/Clock";
@@ -66,6 +68,14 @@ const busyTurnPrefix = (providerThreadId: ProviderThreadId) => `${providerThread
68
const busyTurnKey = (providerThreadId: ProviderThreadId, runOrdinal: number) =>
69
`${busyTurnPrefix(providerThreadId)}${runOrdinal}`;
70
const UNLOAD_THREAD_TIMEOUT_MS = 10 * 1000;
71
+// Providers can announce a retry and never resume. Longer than any adapter's own stall handling.
72
+const PROVIDER_RETRY_STALL_TIMEOUT = Duration.minutes(15);
73
+type ProviderErrorItem = Extract<OrchestrationV2TurnItem, { readonly type: "error" }>;
74
+type ProviderRetryItem = ProviderErrorItem & {
75
+ readonly providerTurnId: ProviderTurnId;
76
+ readonly providerThreadId: ProviderThreadId;
77
+ readonly retry: NonNullable<ProviderErrorItem["retry"]>;
78
+};
79
/** Suggestions outlive their sessions, so keep only the most recently updated threads. */
80
const MAX_PROMPT_SUGGESTIONS = 256;
81
const WORKING_RUN_STATUSES: ReadonlySet<string> = new Set([
@@ -2203,21 +2213,198 @@ export const layerWithOptions = (
2213
),
2214
);
2215
2216
+ const makeRetryStallWatch = (entry: LiveSessionEntry) => {
2217
+ const runningRetries = new Map<ProviderTurnId, ProviderRetryItem>();
2218
+ const stalledRetries = new Map<ProviderTurnId, ProviderRetryItem>();
2219
+ const stallTimers = new Map<ProviderTurnId, Fiber.Fiber<void>>();
2220
+ // Claude and Pi keep the retry row running while a recovered reply streams, so any
2221
+ // event on the retry's thread counts as the provider being alive.
2222
+ const activeSinceCheck = new Set<ProviderTurnId>();
2223
+ const noteThreadActivity = (event: ProviderAdapter.ProviderAdapterV2Event) => {
2224
+ if (runningRetries.size === 0) return;
2225
+ const threadId =
2226
+ event.type === "turn_item.updated"
2227
+ ? event.turnItem.threadId
2228
+ : event.type === "message.updated"
2229
+ ? event.message.threadId
2230
+ : event.type === "node.updated"
2231
+ ? event.node.threadId
2232
+ : event.type === "plan.updated"
2233
+ ? event.plan.threadId
2234
+ : undefined;
2235
+ if (threadId === undefined) return;
2236
+ for (const item of runningRetries.values()) {
2237
+ if (item.threadId === threadId) activeSinceCheck.add(item.providerTurnId);
2238
+ }
2239
+ };
2240
+ // Stops the stalled turn; its interrupted terminal is then reported as the retry's failure.
2241
+ const stopStalledRetry = Effect.fn("ProviderSessionManager.stopStalledRetry")(
2242
+ function* (item: ProviderRetryItem) {
2243
+ if (runningRetries.get(item.providerTurnId) !== item) return;
2244
+ // The terminal this stop causes must not interrupt the stop itself.
2245
+ stallTimers.delete(item.providerTurnId);
2246
+ const current = (yield* Ref.get(sessions)).get(
2247
+ sessionKey(entry.runtime.providerSessionId),
2248
+ );
2249
+ if (current?.runtime !== entry.runtime) return;
2250
+ const { providerThread } = yield* projectionStore.getProviderControlContext(
2251
+ item.threadId,
2252
+ { providerThreadId: item.providerThreadId, providerTurnId: item.providerTurnId },
2253
+ );
2254
+ if (providerThread === undefined || runningRetries.get(item.providerTurnId) !== item)
2255
+ return;
2256
+ stalledRetries.set(item.providerTurnId, item);
2257
+ yield* entry.exposedRuntime
2258
+ .interruptTurn({ providerThread, providerTurnId: item.providerTurnId })
2259
+ .pipe(
2260
+ // A slow stop can still end the turn later, so that terminal keeps the retry failure.
2261
+ Effect.catchTags({
2262
+ ProviderAdapterInterruptTimeoutError: (cause) =>
2263
+ Effect.logWarning("orchestration-v2.provider-retry-stall-stop-timed-out", {
2264
+ providerSessionId: entry.runtime.providerSessionId,
2265
+ cause,
2266
+ }),
2267
+ }),
2268
+ );
2269
+ },
2270
+ (effect, item) =>
2271
+ effect.pipe(
2272
+ Effect.catchCause((cause) =>
2273
+ Effect.sync(() => stalledRetries.delete(item.providerTurnId)).pipe(
2274
+ Effect.andThen(
2275
+ Effect.logWarning("orchestration-v2.provider-retry-stall-stop-failed", {
2276
+ providerSessionId: entry.runtime.providerSessionId,
2277
+ cause,
2278
+ }),
2279
+ ),
2280
+ ),
2281
+ ),
2282
+ ),
2283
+ );
2284
+ const stopStallTimer = (providerTurnId: ProviderTurnId) => {
2285
+ const fiber = stallTimers.get(providerTurnId);
2286
+ stallTimers.delete(providerTurnId);
2287
+ activeSinceCheck.delete(providerTurnId);
2288
+ return fiber === undefined ? Effect.void : Fiber.interrupt(fiber);
2289
+ };
2290
+ const trackProviderRetry = (
2291
+ event: ProviderAdapter.ProviderAdapterV2Event,
2292
+ ): {
2293
+ readonly event: ProviderAdapter.ProviderAdapterV2Event;
2294
+ readonly armed?: ProviderRetryItem;
2295
+ readonly settled?: ProviderTurnId;
2296
+ } => {
2297
+ noteThreadActivity(event);
2298
+ if (event.type === "turn_item.updated") {
2299
+ const item = event.turnItem;
2300
+ if (
2301
+ item.type !== "error" ||
2302
+ item.retry === undefined ||
2303
+ item.providerTurnId === null ||
2304
+ item.providerThreadId === null
2305
+ )
2306
+ return { event };
2307
+ const retryItem = {
2308
+ ...item,
2309
+ providerTurnId: item.providerTurnId,
2310
+ providerThreadId: item.providerThreadId,
2311
+ retry: item.retry,
2312
+ };
2313
+ if (item.status === "running") {
2314
+ runningRetries.set(retryItem.providerTurnId, retryItem);
2315
+ return { event, armed: retryItem };
2316
+ }
2317
+ // A stall marker is set only once the stop is sent, so only the terminal clears it.
2318
+ runningRetries.delete(retryItem.providerTurnId);
2319
+ return { event, settled: retryItem.providerTurnId };
2320
+ }
2321
+ if (event.type === "provider_turn.updated") {
2322
+ if (event.providerTurn.status === "running") return { event };
2323
+ runningRetries.delete(event.providerTurn.id);
2324
+ const settled = event.providerTurn.id;
2325
+ return stalledRetries.has(settled) &&
2326
+ (event.providerTurn.status === "interrupted" ||
2327
+ event.providerTurn.status === "cancelled")
2328
+ ? {
2329
+ event: { ...event, providerTurn: { ...event.providerTurn, status: "failed" } },
2330
+ settled,
2331
+ }
2332
+ : { event, settled };
2333
+ }
2334
+ if (event.type !== "turn.terminal") return { event };
2335
+ runningRetries.delete(event.providerTurnId);
2336
+ const stalled = stalledRetries.get(event.providerTurnId);
2337
+ stalledRetries.delete(event.providerTurnId);
2338
+ const settled = event.providerTurnId;
2339
+ if (
2340
+ stalled === undefined ||
2341
+ (event.status !== "interrupted" && event.status !== "cancelled")
2342
+ )
2343
+ return { event, settled };
2344
+ return {
2345
+ settled,
2346
+ event: {
2347
+ type: "turn.terminal",
2348
+ driver: event.driver,
2349
+ providerThreadId: event.providerThreadId,
2350
+ providerTurnId: event.providerTurnId,
2351
+ runOrdinal: event.runOrdinal,
2352
+ failureItemOrdinal: stalled.ordinal,
2353
+ status: "failed",
2354
+ failure: { ...stalled.failure, retryable: false },
2355
+ retry: stalled.retry,
2356
+ ...(stalled.startedAt === null ? {} : { retryStartedAt: stalled.startedAt }),
2357
+ threadDisposition: event.threadDisposition,
2358
+ },
2359
+ };
2360
+ };
2361
+ return {
2362
+ track: trackProviderRetry,
2363
+ // Runs for every provider event; spans would be per-event noise. Timers are children
2364
+ // of the pump fiber, so a released session takes them down.
2365
+ update: Effect.fnUntraced(function* (tracked: ReturnType<typeof trackProviderRetry>) {
2366
+ const item = tracked.armed;
2367
+ if (item === undefined) {
2368
+ if (tracked.settled !== undefined) yield* stopStallTimer(tracked.settled);
2369
+ return;
2370
+ }
2371
+ yield* stopStallTimer(item.providerTurnId);
2372
+ const window = Duration.sum(
2373
+ PROVIDER_RETRY_STALL_TIMEOUT,
2374
+ Duration.millis(item.retry.retryDelayMs ?? 0),
2375
+ );
2376
+ const timer = yield* Effect.gen(function* () {
2377
+ // Only a thread that stayed silent for a whole window has stalled.
2378
+ do {
2379
+ activeSinceCheck.delete(item.providerTurnId);
2380
+ yield* Effect.sleep(window);
2381
+ } while (activeSinceCheck.has(item.providerTurnId));
2382
+ yield* stopStalledRetry(item);
2383
+ }).pipe(Effect.forkChild({ startImmediately: true }));
2384
+ stallTimers.set(item.providerTurnId, timer);
2385
+ }),
2386
+ };
2387
+ };
2388
+
2389
const startEventPump = (entry: LiveSessionEntry) => {
2390
let stoppedByProvider = false;
2391
let lastSessionStatus = entry.runtime.providerSession.status;
2392
+ const retryStalls = makeRetryStallWatch(entry);
2393
return entry.runtime.events.pipe(
2210
- Stream.runForEach((event) => {
2394
+ Stream.runForEach((nativeEvent) => {
2395
if (shutdownSignal.received) return Effect.void;
2396
// Live output bypasses persistence and subscribers so floods cannot delay domain events.
2213
- if (event.type === "command_output.delta") {
2214
- return commandOutput.append(event);
2397
+ if (nativeEvent.type === "command_output.delta") {
2398
+ return commandOutput.append(nativeEvent);
2399
}
2216
- if (event.type === "prompt_suggestion.updated") {
2400
+ if (nativeEvent.type === "prompt_suggestion.updated") {
2401
return SubscriptionRef.update(promptSuggestionStore, (current) => {
2402
const next = new Map(current);
2219
- next.delete(event.threadId);
2220
- next.set(event.threadId, { runId: event.runId, text: event.text });
2403
+ next.delete(nativeEvent.threadId);
2404
+ next.set(nativeEvent.threadId, {
2405
+ runId: nativeEvent.runId,
2406
+ text: nativeEvent.text,
2407
+ });
2408
for (const threadId of next.keys()) {
2409
if (next.size <= MAX_PROMPT_SUGGESTIONS) break;
2410
next.delete(threadId);
@@ -2225,6 +2412,9 @@ export const layerWithOptions = (
2412
return next;
2413
});
2414
}
2415
+ const tracked = retryStalls.track(nativeEvent);
2416
+ const event = tracked.event;
2417
+ const armRetryStall = retryStalls.update(tracked);
2418
const releaseLiveOutput =
2419
event.type === "turn_item.updated" &&
2420
event.turnItem.type === "command_execution" &&
@@ -2254,6 +2444,7 @@ export const layerWithOptions = (
2444
? Effect.void
2445
: touchActivity(entry.runtime.providerSessionId, { providerActivity: true }),
2446
).pipe(
2447
+ Effect.andThen(armRetryStall),
2448
Effect.andThen(
2449
event.type === "provider_session.updated"
2450
? persistProviderSessionUpdate(entry, event)
packages/provider-pi/src/server/adapter.test.ts
+6
-1
@@ -4744,7 +4744,11 @@ describe("PiAdapterV2", () => {
4744
const running = yield* takeEvent(
4745
(event) => event.type === "turn_item.updated" && event.turnItem.type === "error",
4746
);
4747
- yield* fake.emit({ type: "auto_retry_end", success: true, attempt: 1 });
4747
+ // Streamed tool input settles the retry; Pi sends auto_retry_end only after the message ends.
4748
+ yield* fake.emit({
4749
+ type: "message_update",
4750
+ assistantMessageEvent: { type: "toolcall_delta", contentIndex: 0, delta: "{" },
4751
+ });
4752
const recovered = yield* takeEvent(
4753
(event) => event.type === "turn_item.updated" && event.turnItem.type === "error",
4754
);
@@ -4757,6 +4761,7 @@ describe("PiAdapterV2", () => {
4761
recovered.turnItem.status === "completed" &&
4762
recovered.turnItem.title === "Provider recovered",
4763
);
4764
+ yield* fake.emit({ type: "auto_retry_end", success: true, attempt: 1 });
4765
4766
yield* fake.emit({ type: "agent_settled" });
4767
const terminal = yield* takeEvent((event) => event.type === "turn.terminal");
packages/provider-pi/src/server/adapter.ts
+6
@@ -1967,6 +1967,12 @@ export const makePiAdapterV2 = Effect.fn("makePiAdapterV2")(function* (
1967
case "message_update": {
1968
if (turn === null) return;
1969
turn.sawAgentActivity = true;
1970
+ // Pi sends auto_retry_end only after the message ends; streamed output is recovery.
1971
+ if (turn.activeProviderRetry !== null) {
1972
+ const recoveredAt = yield* DateTime.now;
1973
+ yield* emitProviderRetry(turn, turn.activeProviderRetry, "completed", recoveredAt);
1974
+ turn.activeProviderRetry = null;
1975
+ }
1976
yield* reportLiveUsage(turn, event["usage"]);
1977
const delta = event["assistantMessageEvent"];
1978
const deltaType = recordString(delta, "type");