The best way to control Claude Code, Codex and more from any computer or phone
fix(server): retry Codex capacity failures with empty input (#25611)
Co-authored-by: maria-rcks <maria@kuuro.net>
Yash Singh committed
Oct 27, 2026 at 14:36 UTC
740de0e642d630a08e37d0780fa99c178c27808e
4 files changed
+1124
-69
apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts
+779
@@ -2214,6 +2214,436 @@ describe("CodexAdapterV2 post-settle continuation", () => {
2214
),
2215
);
2216
2217
+ it.effect.each([
2218
+ "recover",
2219
+ "completion-only",
2220
+ "exhaust",
2221
+ "stop",
2222
+ "stop-continued",
2223
+ "stop-starting",
2224
+ "started-first",
2225
+ "completed-first",
2226
+ "new-run",
2227
+ "steer",
2228
+ "start-fails",
2229
+ ] as const)("handles terminal Codex capacity failures when retries %s", (outcome) =>
2230
+ Effect.scoped(
2231
+ Effect.gen(function* () {
2232
+ const scenario = `capacity-${outcome}`;
2233
+ const nativeThreadId = `native-${scenario}`;
2234
+ const nativeTurnId = (attempt: number) => `${nativeThreadId}-${attempt}`;
2235
+ const prompt = "Continue the task.";
2236
+ const capacityError = {
2237
+ message: "Selected model is at capacity. Please try a different model.",
2238
+ codexErrorInfo: "serverOverloaded",
2239
+ };
2240
+ const failureEntries = (attempt: number): Array<CodexReplay.CodexAppServerReplayEntry> => [
2241
+ {
2242
+ type: "emit_inbound",
2243
+ frame: {
2244
+ method: "error",
2245
+ params: {
2246
+ threadId: nativeThreadId,
2247
+ turnId: nativeTurnId(attempt),
2248
+ willRetry: false,
2249
+ error: capacityError,
2250
+ },
2251
+ },
2252
+ },
2253
+ {
2254
+ type: "emit_inbound",
2255
+ frame: {
2256
+ method: "turn/completed",
2257
+ params: {
2258
+ threadId: nativeThreadId,
2259
+ turn: {
2260
+ ...makeCodexReplayTurn({ id: nativeTurnId(attempt), status: "failed" }),
2261
+ error: capacityError,
2262
+ },
2263
+ },
2264
+ },
2265
+ },
2266
+ ];
2267
+ const entries = codexReplayPreamble({
2268
+ nativeThreadId,
2269
+ nativeTurnId: nativeTurnId(0),
2270
+ prompt,
2271
+ });
2272
+ entries.push(
2273
+ ...(outcome === "completion-only" ? failureEntries(0).slice(1) : failureEntries(0)),
2274
+ );
2275
+ if (outcome === "completion-only") entries.push(...failureEntries(0).slice(1));
2276
+ const retries = outcome === "exhaust" ? 5 : outcome === "stop" ? 0 : 1;
2277
+ const steeredText = "Use the smaller change.";
2278
+ for (let attempt = 1; attempt <= retries; attempt++) {
2279
+ const start = codexReplayPreamble({
2280
+ nativeThreadId,
2281
+ nativeTurnId: nativeTurnId(attempt),
2282
+ prompt,
2283
+ }).slice(-3);
2284
+ if (
2285
+ outcome === "started-first" ||
2286
+ outcome === "stop-starting" ||
2287
+ outcome === "completed-first" ||
2288
+ outcome === "new-run"
2289
+ ) {
2290
+ start.splice(1, 0, ...start.splice(2, 1));
2291
+ }
2292
+ for (const entry of start) {
2293
+ if (entry.type === "expect_outbound") {
2294
+ entries.push({
2295
+ ...entry,
2296
+ frame: {
2297
+ id: 3 + attempt,
2298
+ method: "turn/start",
2299
+ params: {
2300
+ threadId: nativeThreadId,
2301
+ input: [],
2302
+ cwd: "/workspace",
2303
+ model: "gpt-5.4",
2304
+ approvalPolicy: "never",
2305
+ approvalsReviewer: "user",
2306
+ sandboxPolicy: { type: "dangerFullAccess" },
2307
+ summary: "detailed",
2308
+ collaborationMode: { mode: "default", settings: { model: "gpt-5.4" } },
2309
+ },
2310
+ },
2311
+ });
2312
+ } else if (entry.type === "emit_inbound" && entry.label === "turn/start") {
2313
+ if (outcome === "completed-first" || outcome === "new-run")
2314
+ entries.push({
2315
+ type: "emit_inbound",
2316
+ frame: {
2317
+ method: "turn/completed",
2318
+ params: {
2319
+ threadId: nativeThreadId,
2320
+ turn: makeCodexReplayTurn({ id: nativeTurnId(attempt), status: "completed" }),
2321
+ },
2322
+ },
2323
+ });
2324
+ if (outcome === "new-run") {
2325
+ entries.push(
2326
+ ...codexReplayPreamble({
2327
+ nativeThreadId,
2328
+ nativeTurnId: nativeTurnId(2),
2329
+ prompt: "Next task.",
2330
+ })
2331
+ .slice(-3)
2332
+ .map((next, index) => (index < 2 ? withReplayRequestId(next, 5) : next)),
2333
+ ...failureEntries(2),
2334
+ );
2335
+ }
2336
+ entries.push(
2337
+ outcome === "start-fails"
2338
+ ? {
2339
+ type: "emit_inbound",
2340
+ frame: {
2341
+ id: 3 + attempt,
2342
+ error: { code: -32603, message: "Start failed" },
2343
+ },
2344
+ }
2345
+ : withReplayRequestId(entry, 3 + attempt),
2346
+ );
2347
+ } else if (outcome !== "start-fails") {
2348
+ entries.push(
2349
+ outcome === "stop-starting" && entry.type === "emit_inbound"
2350
+ ? { ...entry, afterMs: 1_000 }
2351
+ : entry,
2352
+ );
2353
+ if (outcome === "stop-starting") {
2354
+ entries.push(
2355
+ {
2356
+ type: "expect_outbound",
2357
+ frame: {
2358
+ id: 5,
2359
+ method: "turn/interrupt",
2360
+ params: { threadId: nativeThreadId, turnId: nativeTurnId(attempt) },
2361
+ },
2362
+ },
2363
+ { type: "emit_inbound", frame: { id: 5, result: {} } },
2364
+ );
2365
+ }
2366
+ }
2367
+ }
2368
+ if (outcome === "exhaust") entries.push(...failureEntries(attempt));
2369
+ if (
2370
+ outcome === "recover" ||
2371
+ outcome === "completion-only" ||
2372
+ outcome === "started-first" ||
2373
+ outcome === "stop-starting"
2374
+ ) {
2375
+ entries.push({
2376
+ type: "emit_inbound",
2377
+ frame: {
2378
+ method: "turn/completed",
2379
+ params: {
2380
+ threadId: nativeThreadId,
2381
+ turn: makeCodexReplayTurn({
2382
+ id: nativeTurnId(attempt),
2383
+ status: outcome === "stop-starting" ? "interrupted" : "completed",
2384
+ }),
2385
+ },
2386
+ },
2387
+ });
2388
+ }
2389
+ if (outcome === "stop-continued" || outcome === "steer") {
2390
+ const steering = outcome === "steer";
2391
+ entries.push(
2392
+ {
2393
+ type: "expect_outbound",
2394
+ frame: {
2395
+ id: 5,
2396
+ method: steering ? "turn/steer" : "turn/interrupt",
2397
+ params: steering
2398
+ ? {
2399
+ threadId: nativeThreadId,
2400
+ expectedTurnId: nativeTurnId(attempt),
2401
+ input: [{ type: "text", text: steeredText }],
2402
+ }
2403
+ : { threadId: nativeThreadId, turnId: nativeTurnId(attempt) },
2404
+ },
2405
+ },
2406
+ {
2407
+ type: "emit_inbound",
2408
+ frame: { id: 5, result: steering ? { turnId: nativeTurnId(attempt) } : {} },
2409
+ },
2410
+ {
2411
+ type: "emit_inbound",
2412
+ frame: {
2413
+ method: "turn/completed",
2414
+ params: {
2415
+ threadId: nativeThreadId,
2416
+ turn: makeCodexReplayTurn({
2417
+ id: nativeTurnId(attempt),
2418
+ status: steering ? "completed" : "interrupted",
2419
+ }),
2420
+ },
2421
+ },
2422
+ },
2423
+ );
2424
+ }
2425
+ }
2426
+ const milestones = yield* Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event>();
2427
+ const starts: Array<unknown> = [];
2428
+ const retryStarted = yield* Deferred.make<void>();
2429
+ const harness = yield* makeCodexReplayHarness(
2430
+ makeCodexReplayTranscript({ scenario, entries }),
2431
+ (event) =>
2432
+ event.type === "turn.terminal" ||
2433
+ (event.type === "turn_item.updated" &&
2434
+ event.turnItem.type === "error" &&
2435
+ event.turnItem.status === "running")
2436
+ ? Queue.offer(milestones, event)
2437
+ : Effect.void,
2438
+ (method, params) =>
2439
+ Effect.gen(function* () {
2440
+ if (method === "turn/start") {
2441
+ starts.push(params);
2442
+ if (starts.length === 2) yield* Deferred.succeed(retryStarted, undefined);
2443
+ }
2444
+ }),
2445
+ );
2446
+ const turnInput = makeCodexTestTurnInput({
2447
+ threadId: harness.threadId,
2448
+ providerThread: harness.providerThread,
2449
+ now: yield* DateTime.now,
2450
+ attemptId: RunAttemptId.make(scenario),
2451
+ text: prompt,
2452
+ });
2453
+ yield* harness.runtime.startTurn(turnInput);
2454
+ for (let attempt = 1; attempt <= Math.max(retries, 1); attempt++) {
2455
+ const milestone = yield* Queue.take(milestones);
2456
+ assert.equal(milestone.type, "turn_item.updated");
2457
+ if (milestone.type !== "turn_item.updated" || milestone.turnItem.type !== "error") return;
2458
+ const providerTurnId = milestone.turnItem.providerTurnId;
2459
+ assert.isNotNull(providerTurnId);
2460
+ if (providerTurnId === null) return;
2461
+ assert.deepEqual(milestone.turnItem.retry, {
2462
+ attempt,
2463
+ maxAttempts: 5,
2464
+ retryDelayMs: Math.min(2_000 * 2 ** (attempt - 1), 30_000),
2465
+ });
2466
+ assert.isEmpty(harness.terminalEvents());
2467
+ if (outcome === "stop") {
2468
+ yield* harness.runtime.interruptTurn({
2469
+ providerThread: harness.providerThread,
2470
+ providerTurnId,
2471
+ });
2472
+ }
2473
+ const steering =
2474
+ outcome === "steer"
2475
+ ? yield* harness.runtime
2476
+ .steerTurn({
2477
+ ...turnInput,
2478
+ providerTurnId,
2479
+ message: { ...turnInput.message, text: steeredText },
2480
+ })
2481
+ .pipe(Effect.forkScoped)
2482
+ : undefined;
2483
+ yield* TestClock.adjust(Math.min(2_000 * 2 ** (attempt - 1), 30_000));
2484
+ if (outcome === "stop-starting") {
2485
+ yield* Deferred.await(retryStarted);
2486
+ const stopping = yield* harness.runtime
2487
+ .interruptTurn({ providerThread: harness.providerThread, providerTurnId })
2488
+ .pipe(Effect.forkScoped);
2489
+ yield* TestClock.adjust("1 second");
2490
+ yield* Fiber.join(stopping);
2491
+ }
2492
+ if (steering !== undefined) yield* Fiber.join(steering);
2493
+ if (outcome === "stop-continued") {
2494
+ yield* harness.until(() =>
2495
+ harness.events.some(
2496
+ (event) =>
2497
+ event.type === "provider_turn.updated" &&
2498
+ event.providerTurn.nativeTurnRef?.nativeId === nativeTurnId(1),
2499
+ ),
2500
+ );
2501
+ yield* harness.runtime.interruptTurn({
2502
+ providerThread: harness.providerThread,
2503
+ providerTurnId,
2504
+ });
2505
+ }
2506
+ }
2507
+ yield* harness.awaitTerminal(1);
2508
+ if (outcome === "new-run") {
2509
+ assert.equal((yield* Queue.take(milestones)).type, "turn.terminal");
2510
+ yield* harness.runtime.startTurn({
2511
+ ...turnInput,
2512
+ attemptId: RunAttemptId.make("capacity-next-task"),
2513
+ message: { ...turnInput.message, text: "Next task." },
2514
+ });
2515
+ const nextRetry = yield* Queue.take(milestones);
2516
+ assert.equal(nextRetry.type, "turn_item.updated");
2517
+ if (nextRetry.type !== "turn_item.updated" || nextRetry.turnItem.type !== "error") return;
2518
+ assert.equal(nextRetry.turnItem.retry?.attempt, 1);
2519
+ return;
2520
+ }
2521
+ const terminal = harness.terminalEvents()[0];
2522
+ assert.equal(
2523
+ terminal?.status,
2524
+ outcome === "recover" ||
2525
+ outcome === "completion-only" ||
2526
+ outcome === "steer" ||
2527
+ outcome === "started-first" ||
2528
+ outcome === "completed-first"
2529
+ ? "completed"
2530
+ : outcome === "stop" || outcome === "stop-continued" || outcome === "stop-starting"
2531
+ ? "interrupted"
2532
+ : "failed",
2533
+ );
2534
+ assert.lengthOf(starts, 1 + retries);
2535
+ assert.lengthOf(harness.terminalEvents(), 1);
2536
+ assert.lengthOf(
2537
+ harness.events.filter(
2538
+ (event) =>
2539
+ event.type === "provider_turn.updated" && event.providerTurn.status === "running",
2540
+ ),
2541
+ outcome === "start-fails" ? 1 : 1 + retries,
2542
+ );
2543
+ const retryItems = harness.events.flatMap((event) =>
2544
+ event.type === "turn_item.updated" && event.turnItem.type === "error"
2545
+ ? [event.turnItem]
2546
+ : [],
2547
+ );
2548
+ assert.isTrue(
2549
+ ["completed", "failed", "interrupted"].includes(retryItems.at(-1)?.status ?? ""),
2550
+ );
2551
+ assert.lengthOf(new Set(retryItems.map((item) => item.id)), 1);
2552
+
2553
+ if (outcome === "recover") {
2554
+ const fs = yield* FileSystem.FileSystem;
2555
+ const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-capacity-retry-" });
2556
+ const localTranscript = yield* decodeReplayTranscriptJson(
2557
+ (yield* encodeReplayTranscriptJson(
2558
+ makeCodexReplayTranscript({ scenario, entries }),
2559
+ )).replaceAll(yield* encodeStringJson("/workspace"), yield* encodeStringJson(cwd)),
2560
+ );
2561
+ const driver = yield* CodexReplay.makeReplayDriver(localTranscript);
2562
+ yield* Effect.gen(function* () {
2563
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
2564
+ const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
2565
+ const threadId = ThreadId.make("thread:capacity-retry");
2566
+ yield* orchestrator.dispatch({
2567
+ type: "thread.create",
2568
+ commandId: CommandId.make("create-capacity-retry"),
2569
+ threadId,
2570
+ projectId: ProjectId.make("project:capacity-retry"),
2571
+ title: "Capacity retry",
2572
+ modelSelection: CODEX_TEST_MODEL_SELECTION,
2573
+ runtimeMode: "full-access",
2574
+ interactionMode: "default",
2575
+ branch: null,
2576
+ worktreePath: cwd,
2577
+ createdBy: "user",
2578
+ creationSource: "web",
2579
+ });
2580
+ const retrying = yield* orchestrator.streamDomainEvents.pipe(
2581
+ Stream.filter(
2582
+ (event) =>
2583
+ event.type === "turn-item.updated" &&
2584
+ event.payload.type === "error" &&
2585
+ event.payload.retry != null &&
2586
+ event.payload.status === "running",
2587
+ ),
2588
+ Stream.runHead,
2589
+ Effect.forkChild({ startImmediately: true }),
2590
+ );
2591
+ yield* orchestrator.dispatch({
2592
+ type: "message.dispatch",
2593
+ commandId: CommandId.make("start-capacity-retry"),
2594
+ threadId,
2595
+ messageId: MessageId.make("message:capacity-retry"),
2596
+ text: prompt,
2597
+ attachments: [],
2598
+ createdBy: "user",
2599
+ creationSource: "web",
2600
+ dispatchMode: { type: "start_immediately" },
2601
+ });
2602
+ yield* worker.drain();
2603
+ yield* Fiber.join(retrying);
2604
+ assert.equal(
2605
+ (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)?.status,
2606
+ "running",
2607
+ );
2608
+ const completed = yield* orchestrator.streamDomainEvents.pipe(
2609
+ Stream.filter(
2610
+ (event) => event.type === "run.updated" && event.payload.status === "waiting",
2611
+ ),
2612
+ Stream.runHead,
2613
+ Effect.forkChild({ startImmediately: true }),
2614
+ );
2615
+ yield* TestClock.adjust("2 seconds");
2616
+ yield* Fiber.join(completed);
2617
+ yield* worker.drain();
2618
+ const projection = yield* orchestrator.getThreadProjection(threadId);
2619
+ assert.equal(projection.runs.at(-1)?.status, "completed");
2620
+ assert.deepEqual(
2621
+ projection.providerTurns.map((turn) => turn.status),
2622
+ ["failed", "completed"],
2623
+ );
2624
+ assert.lengthOf(
2625
+ projection.messages.filter((message) => message.role === "user"),
2626
+ 1,
2627
+ );
2628
+ assert.isNull((yield* Ref.get(driver.state)).failure);
2629
+ }).pipe(
2630
+ Effect.provide(
2631
+ ProviderReplayHarness.layerWithRegistry(
2632
+ { name: "codex-capacity-retry", runtimePolicyOverride: { cwd } },
2633
+ CodexAdapterV2Testkit.layer({ transcript: localTranscript, driver }),
2634
+ { runEffectWorker: false },
2635
+ ),
2636
+ ),
2637
+ );
2638
+ }
2639
+ }).pipe(
2640
+ Effect.provide(
2641
+ Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer),
2642
+ ),
2643
+ ),
2644
+ ),
2645
+ );
2646
+
2647
it.effect("resolves retryable app-server errors on resumed provider activity", () =>
2648
Effect.scoped(
2649
Effect.gen(function* () {
@@ -5007,6 +5437,265 @@ describe("CodexAdapterV2 post-settle continuation", () => {
5437
};
5438
};
5439
5440
+ it.effect.each(["completed", "failed", "interrupted", "starting", "replied-last"] as const)(
5441
+ "handles a late native child after the capacity retry %s",
5442
+ (parentStatus) =>
5443
+ Effect.scoped(
5444
+ Effect.gen(function* () {
5445
+ const retriedTurnId = "native-capacity-recovered-parent";
5446
+ const retrying = yield* Deferred.make<void>();
5447
+ const entries: Array<CodexReplay.CodexAppServerReplayEntry> = [
5448
+ ...lateChildPrefix().spawned,
5449
+ {
5450
+ type: "emit_inbound",
5451
+ frame: {
5452
+ method: "turn/completed",
5453
+ params: {
5454
+ threadId: RESUME_NATIVE_THREAD,
5455
+ turn: {
5456
+ ...makeCodexReplayTurn({ id: RESUME_NATIVE_TURN, status: "failed" }),
5457
+ error: {
5458
+ message: "Selected model is at capacity.",
5459
+ codexErrorInfo: "serverOverloaded",
5460
+ },
5461
+ },
5462
+ },
5463
+ },
5464
+ },
5465
+ {
5466
+ type: "expect_outbound",
5467
+ frame: {
5468
+ id: 4,
5469
+ method: "turn/start",
5470
+ params: {
5471
+ threadId: RESUME_NATIVE_THREAD,
5472
+ input: [],
5473
+ cwd: "/workspace",
5474
+ model: "gpt-5.4",
5475
+ approvalPolicy: "never",
5476
+ approvalsReviewer: "user",
5477
+ sandboxPolicy: { type: "dangerFullAccess" },
5478
+ summary: "detailed",
5479
+ collaborationMode: { mode: "default", settings: { model: "gpt-5.4" } },
5480
+ },
5481
+ },
5482
+ },
5483
+ ...(parentStatus === "starting" ? [childTurnCompleted(RESUME_CHILD_TURN_1)] : []),
5484
+ {
5485
+ type: "emit_inbound",
5486
+ frame: {
5487
+ id: 4,
5488
+ result: {
5489
+ turn: makeCodexReplayTurn({ id: retriedTurnId, status: "inProgress" }),
5490
+ },
5491
+ },
5492
+ },
5493
+ {
5494
+ type: "emit_inbound",
5495
+ frame: {
5496
+ method: "turn/started",
5497
+ params: {
5498
+ threadId: RESUME_NATIVE_THREAD,
5499
+ turn: makeCodexReplayTurn({ id: retriedTurnId, status: "inProgress" }),
5500
+ },
5501
+ },
5502
+ },
5503
+ {
5504
+ type: "emit_inbound",
5505
+ frame: {
5506
+ method: "turn/completed",
5507
+ params: {
5508
+ threadId: RESUME_NATIVE_THREAD,
5509
+ turn: {
5510
+ ...makeCodexReplayTurn({
5511
+ id: retriedTurnId,
5512
+ status:
5513
+ parentStatus === "starting" || parentStatus === "replied-last"
5514
+ ? "completed"
5515
+ : parentStatus,
5516
+ }),
5517
+ error:
5518
+ parentStatus === "failed"
5519
+ ? { message: "Recovery failed.", codexErrorInfo: "internalServerError" }
5520
+ : null,
5521
+ },
5522
+ },
5523
+ },
5524
+ },
5525
+ childAgentMessage({
5526
+ id: "child-after-capacity",
5527
+ text: "CAPACITY_CHILD_DONE",
5528
+ turnId: RESUME_CHILD_TURN_1,
5529
+ completedAtMs: 1782622480000,
5530
+ afterMs: 60_000,
5531
+ }),
5532
+ childTurnCompleted(RESUME_CHILD_TURN_1),
5533
+ ];
5534
+ if (parentStatus === "replied-last") {
5535
+ entries.push(...entries.splice(lateChildPrefix().spawned.length + 2, 1));
5536
+ }
5537
+ const harness = yield* makeCodexReplayHarness(
5538
+ makeCodexReplayTranscript({ scenario: "capacity-late-child", entries }),
5539
+ (event) =>
5540
+ event.type === "turn_item.updated" &&
5541
+ event.turnItem.type === "error" &&
5542
+ event.turnItem.status === "running"
5543
+ ? Deferred.succeed(retrying, undefined)
5544
+ : Effect.void,
5545
+ );
5546
+ yield* harness.runtime.startTurn(
5547
+ makeCodexTestTurnInput({
5548
+ threadId: harness.threadId,
5549
+ providerThread: harness.providerThread,
5550
+ now: yield* DateTime.now,
5551
+ attemptId: RunAttemptId.make("capacity-late-child"),
5552
+ text: RESUME_PROMPT,
5553
+ }),
5554
+ );
5555
+ yield* Deferred.await(retrying);
5556
+ yield* TestClock.adjust("2 seconds");
5557
+ if (parentStatus === "interrupted") assert.isEmpty(harness.terminalEvents());
5558
+ else {
5559
+ yield* harness.awaitTerminal(1);
5560
+ const terminal = harness.terminalEvents()[0];
5561
+ if (terminal?.status === "failed") assert.isTrue(terminal.childTurnsContinue);
5562
+ }
5563
+ yield* TestClock.adjust("60 seconds");
5564
+ yield* harness.until(
5565
+ () => harness.subagentUpdates().at(-1)?.subagent.status === "completed",
5566
+ );
5567
+ yield* harness.awaitTerminal(1);
5568
+ assert.lengthOf(
5569
+ harness.continuationRequests,
5570
+ parentStatus === "completed" || parentStatus === "replied-last" ? 1 : 0,
5571
+ );
5572
+ if (parentStatus === "completed" || parentStatus === "replied-last")
5573
+ assert.include(harness.continuationRequests[0]?.detail, "CAPACITY_CHILD_DONE");
5574
+ }).pipe(
5575
+ Effect.provide(
5576
+ Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer),
5577
+ ),
5578
+ ),
5579
+ ),
5580
+ );
5581
+
5582
+ it.effect("stops native children when capacity retry startup fails", () =>
5583
+ Effect.scoped(
5584
+ Effect.gen(function* () {
5585
+ const retrying = yield* Deferred.make<void>();
5586
+ const starting = yield* Deferred.make<void>();
5587
+ const release = yield* Deferred.make<void>();
5588
+ const interrupts: unknown[] = [];
5589
+ let starts = 0;
5590
+ const harness = yield* makeCodexReplayHarness(
5591
+ makeCodexReplayTranscript({
5592
+ scenario: "capacity-start-fails-stop-child",
5593
+ entries: [
5594
+ ...lateChildPrefix().spawned,
5595
+ {
5596
+ type: "emit_inbound",
5597
+ frame: {
5598
+ method: "turn/completed",
5599
+ params: {
5600
+ threadId: RESUME_NATIVE_THREAD,
5601
+ turn: {
5602
+ ...makeCodexReplayTurn({ id: RESUME_NATIVE_TURN, status: "failed" }),
5603
+ error: { message: "At capacity", codexErrorInfo: "serverOverloaded" },
5604
+ },
5605
+ },
5606
+ },
5607
+ },
5608
+ {
5609
+ type: "expect_outbound",
5610
+ frame: {
5611
+ id: 4,
5612
+ method: "turn/start",
5613
+ params: {
5614
+ threadId: RESUME_NATIVE_THREAD,
5615
+ input: [],
5616
+ cwd: "/workspace",
5617
+ model: "gpt-5.4",
5618
+ approvalPolicy: "never",
5619
+ approvalsReviewer: "user",
5620
+ sandboxPolicy: { type: "dangerFullAccess" },
5621
+ summary: "detailed",
5622
+ collaborationMode: { mode: "default", settings: { model: "gpt-5.4" } },
5623
+ },
5624
+ },
5625
+ },
5626
+ {
5627
+ type: "emit_inbound",
5628
+ frame: { id: 4, error: { code: -32603, message: "Start failed" } },
5629
+ },
5630
+ {
5631
+ type: "expect_outbound",
5632
+ frame: {
5633
+ id: 5,
5634
+ method: "turn/interrupt",
5635
+ params: { threadId: RESUME_CHILD_THREAD, turnId: RESUME_CHILD_TURN_1 },
5636
+ },
5637
+ },
5638
+ { type: "emit_inbound", frame: { id: 5, result: {} } },
5639
+ {
5640
+ type: "emit_inbound",
5641
+ frame: {
5642
+ method: "turn/completed",
5643
+ params: {
5644
+ threadId: RESUME_CHILD_THREAD,
5645
+ turn: makeCodexReplayTurn({ id: RESUME_CHILD_TURN_1, status: "interrupted" }),
5646
+ },
5647
+ },
5648
+ },
5649
+ ],
5650
+ }),
5651
+ (event) =>
5652
+ event.type === "turn_item.updated" &&
5653
+ event.turnItem.type === "error" &&
5654
+ event.turnItem.status === "running"
5655
+ ? Deferred.succeed(retrying, undefined)
5656
+ : Effect.void,
5657
+ (method, params) =>
5658
+ Effect.gen(function* () {
5659
+ if (method === "turn/interrupt") interrupts.push(params);
5660
+ if (method === "turn/start" && ++starts === 2) {
5661
+ yield* Deferred.succeed(starting, undefined);
5662
+ yield* Deferred.await(release);
5663
+ }
5664
+ }),
5665
+ );
5666
+ yield* harness.runtime.startTurn(
5667
+ makeCodexTestTurnInput({
5668
+ threadId: harness.threadId,
5669
+ providerThread: harness.providerThread,
5670
+ now: yield* DateTime.now,
5671
+ attemptId: RunAttemptId.make("capacity-start-fails-stop-child"),
5672
+ text: RESUME_PROMPT,
5673
+ }),
5674
+ );
5675
+ yield* Deferred.await(retrying);
5676
+ const root = harness.events.find((event) => event.type === "provider_turn.updated");
5677
+ assert.equal(root?.type, "provider_turn.updated");
5678
+ if (root?.type !== "provider_turn.updated") return;
5679
+ yield* TestClock.adjust("2 seconds");
5680
+ yield* Deferred.await(starting);
5681
+ const stopping = yield* harness.runtime
5682
+ .interruptTurn({
5683
+ providerThread: harness.providerThread,
5684
+ providerTurnId: root.providerTurn.id,
5685
+ })
5686
+ .pipe(Effect.forkChild({ startImmediately: true }));
5687
+ yield* Deferred.succeed(release, undefined);
5688
+ yield* Fiber.join(stopping);
5689
+ assert.lengthOf(interrupts, 1);
5690
+ assert.equal(harness.subagentUpdates().at(-1)?.subagent.status, "interrupted");
5691
+ }).pipe(
5692
+ Effect.provide(
5693
+ Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer),
5694
+ ),
5695
+ ),
5696
+ ),
5697
+ );
5698
+
5699
it.effect("names a late child spawned without a prompt in its wake", () =>
5700
Effect.scoped(
5701
Effect.gen(function* () {
@@ -6434,6 +7123,96 @@ describe("CodexAdapterV2 post-settle continuation", () => {
7123
),
7124
);
7125
7126
+ it.effect("Stop prevents a capacity retry while pausing its goal", () =>
7127
+ Effect.gen(function* () {
7128
+ const nativeTurnId = "goal-capacity-stop";
7129
+ const prompt = "Keep going";
7130
+ const retrying = yield* Deferred.make<void>();
7131
+ const pausing = yield* Deferred.make<void>();
7132
+ const requests: Array<string> = [];
7133
+ const harness = yield* makeCodexReplayHarness(
7134
+ makeCodexReplayTranscript({
7135
+ scenario: "goal-capacity-stop",
7136
+ entries: [
7137
+ ...codexReplayPreamble({ nativeThreadId, nativeTurnId, prompt }),
7138
+ notification("goal active", "thread/goal/updated", {
7139
+ threadId: nativeThreadId,
7140
+ turnId: nativeTurnId,
7141
+ goal: codexGoal("active", 10),
7142
+ }),
7143
+ notification("capacity failure", "turn/completed", {
7144
+ threadId: nativeThreadId,
7145
+ turn: {
7146
+ ...makeCodexReplayTurn({ id: nativeTurnId, status: "failed" }),
7147
+ error: {
7148
+ message: "Selected model is at capacity.",
7149
+ codexErrorInfo: "serverOverloaded",
7150
+ },
7151
+ },
7152
+ }),
7153
+ {
7154
+ type: "expect_outbound",
7155
+ frame: {
7156
+ id: 4,
7157
+ method: "thread/goal/set",
7158
+ params: { threadId: nativeThreadId, status: "paused" },
7159
+ },
7160
+ },
7161
+ {
7162
+ type: "emit_inbound",
7163
+ afterMs: 3_000,
7164
+ frame: { id: 4, result: { goal: codexGoal("paused", 10) } },
7165
+ },
7166
+ ],
7167
+ }),
7168
+ (event) =>
7169
+ event.type === "turn_item.updated" &&
7170
+ event.turnItem.type === "error" &&
7171
+ event.turnItem.status === "running"
7172
+ ? Deferred.succeed(retrying, undefined)
7173
+ : Effect.void,
7174
+ (method) =>
7175
+ Effect.gen(function* () {
7176
+ requests.push(method);
7177
+ if (method === "thread/goal/set") yield* Deferred.succeed(pausing, undefined);
7178
+ }),
7179
+ );
7180
+ yield* harness.runtime.startTurn(
7181
+ makeCodexTestTurnInput({
7182
+ threadId: harness.threadId,
7183
+ providerThread: harness.providerThread,
7184
+ now: yield* DateTime.now,
7185
+ attemptId: RunAttemptId.make("goal-capacity-stop"),
7186
+ text: prompt,
7187
+ }),
7188
+ );
7189
+ yield* Deferred.await(retrying);
7190
+ const stopping = yield* harness.runtime
7191
+ .interruptTurn({
7192
+ providerThread: harness.providerThread,
7193
+ providerTurnId: (yield* IdAllocator.IdAllocatorV2).derive.providerTurn({
7194
+ driver: CodexAdapterV2.CODEX_DRIVER_KIND,
7195
+ nativeTurnId,
7196
+ }),
7197
+ })
7198
+ .pipe(Effect.forkScoped);
7199
+ yield* Deferred.await(pausing);
7200
+ yield* TestClock.adjust("3 seconds");
7201
+ assert.lengthOf(
7202
+ requests.filter((method) => method === "turn/start"),
7203
+ 1,
7204
+ );
7205
+ yield* Fiber.join(stopping);
7206
+ yield* harness.awaitTerminal(1);
7207
+ assert.equal(harness.terminalEvents()[0]?.status, "interrupted");
7208
+ }).pipe(
7209
+ Effect.scoped,
7210
+ Effect.provide(
7211
+ Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer),
7212
+ ),
7213
+ ),
7214
+ );
7215
+
7216
it.effect("pauses an active goal before Stop interrupts its turn", () =>
7217
Effect.gen(function* () {
7218
const prompt = "Keep going";
apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts
+315
-67
@@ -86,6 +86,7 @@ import * as Deferred from "effect/Deferred";
86
import * as Effect from "effect/Effect";
87
import * as Fiber from "effect/Fiber";
88
import * as FileSystem from "effect/FileSystem";
89
+import * as Function from "effect/Function";
90
import * as Layer from "effect/Layer";
91
import * as Option from "effect/Option";
92
import * as Path from "effect/Path";
@@ -152,6 +153,7 @@ import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
153
import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
154
155
const CODEX_PROVIDER = ProviderDriverKind.make("codex");
156
+const CODEX_CAPACITY_MAX_RETRIES = 5;
157
export const CODEX_DRIVER_KIND = CODEX_PROVIDER;
158
export const CODEX_DEFAULT_INSTANCE_ID = defaultInstanceIdForDriver(CODEX_DRIVER_KIND);
159
@@ -1109,6 +1111,8 @@ function withCodexManagedFailure(
1111
}
1112
1113
interface ActiveCodexTurnContext {
1114
+ capacityRetryAttempt?: number;
1115
+ capacityRetryPending?: Deferred.Deferred<ActiveCodexTurnContext | undefined>;
1116
closedCommandItemIds?: ReadonlySet<string>;
1117
latestProviderFailure?: {
1118
readonly nativeMessage: string;
@@ -1141,6 +1145,7 @@ interface ActiveCodexTurnContext {
1145
}
1146
1147
interface ActiveCodexProviderRetry {
1148
+ readonly context?: ActiveCodexTurnContext;
1149
readonly nativeMessage: string;
1150
readonly nativeCode: string | null;
1151
readonly retry: OrchestrationV2ProviderRetry;
@@ -1149,6 +1154,30 @@ interface ActiveCodexProviderRetry {
1154
readonly itemOrdinal: number;
1155
}
1156
1157
+function codexRetryTurnItem(
1158
+ activeContext: ActiveCodexTurnContext,
1159
+ state: ActiveCodexProviderRetry,
1160
+ status: Parameters<typeof makeProviderRetryTurnItem>[0]["status"],
1161
+ updatedAt: DateTime.Utc,
1162
+) {
1163
+ // A continuation keeps the retry row attached to the native turn that failed.
1164
+ const context = state.context ?? activeContext;
1165
+ return makeProviderRetryTurnItem({
1166
+ driver: CODEX_PROVIDER,
1167
+ threadId: context.projectionThreadId,
1168
+ runId: context.projectionRunId,
1169
+ nodeId: context.providerNodeId,
1170
+ providerThreadId: context.providerThread.id,
1171
+ providerTurnId: context.providerTurnId,
1172
+ itemOrdinal: state.itemOrdinal,
1173
+ failure: state.failure,
1174
+ retry: state.retry,
1175
+ status,
1176
+ startedAt: state.startedAt,
1177
+ updatedAt,
1178
+ });
1179
+}
1180
+
1181
interface TrackedRunningCommandItem {
1182
readonly id: string;
1183
readonly command: string;
@@ -1815,12 +1844,23 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
1844
const goalsByNativeThread = new Map<string, OrchestrationV2ProviderGoal | null>();
1845
const rootProviderThreads = new Map<string, OrchestrationV2ProviderThread>();
1846
const goalHolds = new Map<string, CodexGoalHold>();
1818
- const goalRuns = new Map<ProviderTurnId, Array<ActiveCodexTurnContext>>();
1847
+ const continuedRuns = new Map<ProviderTurnId, Array<ActiveCodexTurnContext>>();
1848
+ const capacityContinuations = new Map<
1849
+ string,
1850
+ { readonly context: ActiveCodexTurnContext; readonly retry: ActiveCodexProviderRetry }
1851
+ >();
1852
const goalActivations = new Map<string, { stopped: boolean }>();
1853
// Held runs must not complete while Stop resolves its target.
1854
const goalStops = new Map<string, number>();
1822
- const latestGoalTurnId = (providerTurnId: ProviderTurnId) =>
1823
- goalRuns.get(providerTurnId)?.at(-1)?.providerTurnId ?? providerTurnId;
1855
+ const latestRunTurnId = (providerTurnId: ProviderTurnId) =>
1856
+ continuedRuns.get(providerTurnId)?.at(-1)?.providerTurnId ?? providerTurnId;
1857
+ const isContinuedRunDescendant = (
1858
+ candidate: ActiveCodexTurnContext,
1859
+ context: ActiveCodexTurnContext,
1860
+ ) =>
1861
+ (continuedRuns.get(context.providerTurnId) ?? [context]).some((ancestor) =>
1862
+ isDescendantCodexTurn(candidate, ancestor),
1863
+ );
1864
1865
const emitProviderEvent = (event: ProviderAdapter.ProviderAdapterV2Event) =>
1866
Queue.offer(events, event).pipe(Effect.asVoid);
@@ -1882,20 +1922,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
1922
yield* emitProviderEvent({
1923
type: "turn_item.updated",
1924
driver: CODEX_PROVIDER,
1885
- turnItem: makeProviderRetryTurnItem({
1886
- driver: CODEX_PROVIDER,
1887
- threadId: context.projectionThreadId,
1888
- runId: context.projectionRunId,
1889
- nodeId: context.providerNodeId,
1890
- providerThreadId: context.providerThread.id,
1891
- providerTurnId: context.providerTurnId,
1892
- itemOrdinal: providerRetry.itemOrdinal,
1893
- failure: providerRetry.failure,
1894
- retry: providerRetry.retry,
1895
- status: "completed",
1896
- startedAt: providerRetry.startedAt,
1897
- updatedAt,
1898
- }),
1925
+ turnItem: codexRetryTurnItem(context, providerRetry, "completed", updatedAt),
1926
});
1927
});
1928
@@ -1934,6 +1961,24 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
1961
fileChangeKindsByItem: new Map(),
1962
runningCompactionItemIds: new Set(),
1963
};
1964
+ const nativeThreadId = input.turnInput.providerThread.nativeThreadRef?.nativeId;
1965
+ const continuation =
1966
+ nativeThreadId == null ? undefined : capacityContinuations.get(nativeThreadId);
1967
+ if (continuation !== undefined) {
1968
+ context.capacityRetryAttempt = continuation.context.capacityRetryAttempt ?? 0;
1969
+ yield* Ref.update(providerRetries, (current) =>
1970
+ new Map(current).set(providerTurnId, continuation.retry),
1971
+ );
1972
+ const turns = continuedRuns.get(continuation.context.providerTurnId) ?? [
1973
+ continuation.context,
1974
+ ];
1975
+ turns.push(context);
1976
+ continuedRuns.set(continuation.context.providerTurnId, turns);
1977
+ continuedRuns.set(providerTurnId, turns);
1978
+ if (continuation.context.capacityRetryPending !== undefined) {
1979
+ yield* Deferred.succeed(continuation.context.capacityRetryPending, context);
1980
+ }
1981
+ }
1982
yield* Ref.update(limitedTurnItems, (current) => {
1983
const next = new Map(current);
1984
next.delete(context.providerThread.id);
@@ -1946,6 +1991,9 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
1991
updated.set(input.nativeTurnId, context);
1992
return updated;
1993
});
1994
+ if (continuation !== undefined && nativeThreadId != null) {
1995
+ capacityContinuations.delete(nativeThreadId);
1996
+ }
1997
yield* emitProviderEvent({
1998
type: "provider_turn.updated",
1999
driver: CODEX_PROVIDER,
@@ -4380,10 +4428,12 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
4428
nativeTurnId: payload.turn.id,
4429
startedAt: codexTimestamp(payload.turn.startedAt),
4430
});
4383
- const goalRun = goalRuns.get(goalHold.context.providerTurnId) ?? [goalHold.context];
4431
+ const goalRun = continuedRuns.get(goalHold.context.providerTurnId) ?? [
4432
+ goalHold.context,
4433
+ ];
4434
goalRun.push(next);
4385
- goalRuns.set(goalHold.context.providerTurnId, goalRun);
4386
- goalRuns.set(next.providerTurnId, goalRun);
4435
+ continuedRuns.set(goalHold.context.providerTurnId, goalRun);
4436
+ continuedRuns.set(next.providerTurnId, goalRun);
4437
yield* Deferred.succeed(goalHold.next, next);
4438
return;
4439
}
@@ -4493,6 +4543,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
4543
previous?.itemOrdinal ??
4544
(yield* resolveItemOrdinal(context, `terminal-failure:${context.providerTurnId}`));
4545
const state: ActiveCodexProviderRetry = {
4546
+ ...(previous?.context === undefined ? {} : { context: previous.context }),
4547
nativeMessage: payload.error.message,
4548
nativeCode: code,
4549
retry,
@@ -4508,20 +4559,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
4559
yield* emitProviderEvent({
4560
type: "turn_item.updated",
4561
driver: CODEX_PROVIDER,
4511
- turnItem: makeProviderRetryTurnItem({
4512
- driver: CODEX_PROVIDER,
4513
- threadId: context.projectionThreadId,
4514
- runId: context.projectionRunId,
4515
- nodeId: context.providerNodeId,
4516
- providerThreadId: context.providerThread.id,
4517
- providerTurnId: context.providerTurnId,
4518
- itemOrdinal,
4519
- failure,
4520
- retry,
4521
- status: "running",
4522
- startedAt: state.startedAt,
4523
- updatedAt,
4524
- }),
4562
+ turnItem: codexRetryTurnItem(context, state, "running", updatedAt),
4563
});
4564
}).pipe(Effect.orDie),
4565
);
@@ -5617,8 +5655,16 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
5655
}
5656
: event;
5657
yield* emitProviderEvent(current);
5620
- for (const turn of goalRuns.get(context.providerTurnId) ?? []) {
5621
- goalRuns.delete(turn.providerTurnId);
5658
+ const continued = continuedRuns.get(context.providerTurnId) ?? [context];
5659
+ if (current.status !== "completed") {
5660
+ yield* Ref.update(
5661
+ terminalizedNonCompletedNativeTurns,
5662
+ (nativeTurns) =>
5663
+ new Set([...nativeTurns, ...continued.map((turn) => turn.nativeTurnId)]),
5664
+ );
5665
+ }
5666
+ for (const turn of continued) {
5667
+ continuedRuns.delete(turn.providerTurnId);
5668
}
5669
const nativeThreadId = context.providerThread.nativeThreadRef?.nativeId;
5670
const rootProviderThread =
@@ -5656,7 +5702,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
5702
}) {
5703
const event = yield* makeRootTerminalEvent(input);
5704
const hasActiveDescendants = Array.from((yield* Ref.get(activeTurns)).values()).some(
5659
- (candidate) => isDescendantCodexTurn(candidate, input.context),
5705
+ (candidate) => isContinuedRunDescendant(candidate, input.context),
5706
);
5707
// Failed roots leave native children working and must settle without them.
5708
// Interrupted roots wait for descendants because Stop interrupts the whole lineage.
@@ -5781,7 +5827,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
5827
for (const [nativeTurnId, deferred] of current) {
5828
if (
5829
!activeTurnContexts.some((candidate) =>
5784
- isDescendantCodexTurn(candidate, deferred.context),
5830
+ isContinuedRunDescendant(candidate, deferred.context),
5831
)
5832
) {
5833
updated.delete(nativeTurnId);
@@ -5803,8 +5849,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
5849
readonly completedAt: DateTime.Utc;
5850
readonly failureMessage?: string;
5851
readonly failureErrorInfo?: CodexErrorInfo;
5852
+ readonly continueRun?: boolean;
5853
}) =>
5807
- turnTerminalizationPermit.withPermits(1)(
5854
+ // Capacity continuation already holds the permit across finalization and restart.
5855
+ (input.continueRun ? Function.identity : turnTerminalizationPermit.withPermits(1))(
5856
Effect.gen(function* () {
5857
const current = (yield* Ref.get(activeTurns)).get(input.nativeTurnId);
5858
if (current !== input.context) {
@@ -5822,31 +5870,33 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
5870
});
5871
if (
5872
providerRetry !== undefined &&
5825
- (input.status !== "failed" || input.context.subagent !== null)
5873
+ !input.continueRun &&
5874
+ (providerRetry.context !== undefined ||
5875
+ input.status !== "failed" ||
5876
+ input.context.subagent !== null)
5877
) {
5878
yield* emitProviderEvent({
5879
type: "turn_item.updated",
5880
driver: CODEX_PROVIDER,
5830
- turnItem: makeProviderRetryTurnItem({
5831
- driver: CODEX_PROVIDER,
5832
- threadId: input.context.projectionThreadId,
5833
- runId: input.context.projectionRunId,
5834
- nodeId: input.context.providerNodeId,
5835
- providerThreadId: input.context.providerThread.id,
5836
- providerTurnId: input.context.providerTurnId,
5837
- itemOrdinal: providerRetry.itemOrdinal,
5838
- failure: providerRetry.failure,
5839
- retry: providerRetry.retry,
5840
- status: providerTurnStatusToTerminal(input.status),
5841
- startedAt: providerRetry.startedAt,
5842
- updatedAt: input.completedAt,
5843
- }),
5881
+ turnItem: codexRetryTurnItem(
5882
+ input.context,
5883
+ providerRetry,
5884
+ input.status === "failed" &&
5885
+ providerRetry.context !== input.context &&
5886
+ providerRetry.context !== undefined
5887
+ ? "interrupted"
5888
+ : providerTurnStatusToTerminal(input.status),
5889
+ input.completedAt,
5890
+ ),
5891
});
5892
}
5846
- if (input.status !== "completed") {
5893
+ if (input.status !== "completed" && !input.continueRun) {
5894
yield* Ref.update(terminalizedNonCompletedNativeTurns, (current) => {
5895
const updated = new Set(current);
5849
- updated.add(input.nativeTurnId);
5896
+ for (const turn of continuedRuns.get(input.context.providerTurnId) ?? [
5897
+ input.context,
5898
+ ])
5899
+ updated.add(turn.nativeTurnId);
5900
return updated;
5901
});
5902
}
@@ -5967,11 +6017,14 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
6017
// Only direct children wake an idle root turn; grandchildren report to their
6018
// own parent, and running root turns receive results natively.
6019
const parent = input.context.subagent.parentContext;
6020
+ const parentNativeThreadId = parent.providerThread.nativeThreadRef?.nativeId;
6021
if (
6022
continuationRequests !== undefined &&
6023
parent.subagent === null &&
6024
(input.status === "completed" || input.status === "failed") &&
6025
!(yield* Ref.get(terminalizedNonCompletedNativeTurns)).has(parent.nativeTurnId) &&
6026
+ (parentNativeThreadId == null ||
6027
+ !capacityContinuations.has(parentNativeThreadId)) &&
6028
![...(yield* Ref.get(activeTurns)).values()].some(
6029
(active) =>
6030
active.subagent === null &&
@@ -6017,7 +6070,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
6070
input.completedAt,
6071
input.status === "interrupted" || input.status === "failed",
6072
);
6020
- if (input.context.subagent === null) {
6073
+ if (input.context.subagent === null && !input.continueRun) {
6074
yield* emitOrDeferRootTerminal({
6075
...input,
6076
...(providerRetry === undefined ? {} : { providerRetry }),
@@ -6051,6 +6104,9 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
6104
if (input.context.nativeStartReady !== undefined) {
6105
yield* Deferred.succeed(input.context.nativeStartReady, undefined);
6106
}
6107
+ if (!input.continueRun && input.context.capacityRetryPending !== undefined) {
6108
+ yield* Deferred.succeed(input.context.capacityRetryPending, undefined);
6109
+ }
6110
yield* flushReadyRootTerminals();
6111
if (!retainSettledContext && !interruptInProgress) {
6112
yield* Ref.update(runningCommandItemsByTurn, (current) => {
@@ -6092,6 +6148,141 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
6148
}),
6149
);
6150
6151
+ // Codex can end an overloaded turn without retry advice. Continue its
6152
+ // persisted history with empty input instead of resending the user message.
6153
+ const retryCapacityFailure = Effect.fn("CodexAdapterV2.retryCapacityFailure")(function* (
6154
+ terminal: Parameters<typeof finalizeCodexTurn>[0],
6155
+ ) {
6156
+ const context = terminal.context;
6157
+ const code =
6158
+ terminal.failureErrorInfo == null
6159
+ ? context.latestProviderFailure?.nativeCode
6160
+ : codexErrorInfoCode(terminal.failureErrorInfo);
6161
+ if (
6162
+ terminal.status !== "failed" ||
6163
+ code !== "serverOverloaded" ||
6164
+ context.subagent !== null ||
6165
+ (context.input.message.attachments.length === 0 &&
6166
+ context.input.message.text.trim().toLowerCase() === "/compact")
6167
+ )
6168
+ return false;
6169
+
6170
+ return yield* turnTerminalizationPermit.withPermits(1)(
6171
+ Effect.gen(function* () {
6172
+ if ((yield* Ref.get(activeTurns)).get(terminal.nativeTurnId) !== context) return true;
6173
+ if (context.capacityRetryPending !== undefined) return true;
6174
+ if (
6175
+ (context.capacityRetryAttempt ?? 0) >= CODEX_CAPACITY_MAX_RETRIES ||
6176
+ (yield* Ref.get(interruptingNativeTurns)).has(terminal.nativeTurnId)
6177
+ )
6178
+ return false;
6179
+
6180
+ const attempt = (context.capacityRetryAttempt ?? 0) + 1;
6181
+ context.capacityRetryAttempt = attempt;
6182
+ const pending = yield* Deferred.make<ActiveCodexTurnContext | undefined>();
6183
+ context.capacityRetryPending = pending;
6184
+ const retryDelayMs = Math.min(2_000 * 2 ** (attempt - 1), 30_000);
6185
+ const now = yield* DateTime.now;
6186
+ const previous = (yield* Ref.get(providerRetries)).get(context.providerTurnId);
6187
+ const state: ActiveCodexProviderRetry = {
6188
+ context: previous?.context ?? context,
6189
+ nativeMessage:
6190
+ terminal.failureMessage ??
6191
+ context.latestProviderFailure?.nativeMessage ??
6192
+ "Selected model is at capacity.",
6193
+ nativeCode: code,
6194
+ failure: makeProviderFailure({
6195
+ message:
6196
+ context.latestProviderFailure?.failure.message ?? terminal.failureMessage,
6197
+ code,
6198
+ class: "provider_error",
6199
+ retryable: true,
6200
+ }),
6201
+ retry: { attempt, maxAttempts: CODEX_CAPACITY_MAX_RETRIES, retryDelayMs },
6202
+ itemOrdinal:
6203
+ previous?.itemOrdinal ??
6204
+ (yield* resolveItemOrdinal(
6205
+ context,
6206
+ `terminal-failure:${context.providerTurnId}`,
6207
+ )),
6208
+ startedAt: previous?.startedAt ?? now,
6209
+ };
6210
+ yield* Ref.update(providerRetries, (current) =>
6211
+ new Map(current).set(context.providerTurnId, state),
6212
+ );
6213
+ yield* emitProviderEvent({
6214
+ type: "turn_item.updated",
6215
+ driver: CODEX_PROVIDER,
6216
+ turnItem: codexRetryTurnItem(context, state, "running", now),
6217
+ });
6218
+ yield* Effect.sleep(retryDelayMs).pipe(
6219
+ Effect.andThen(
6220
+ Effect.gen(function* () {
6221
+ const nativeThreadId = yield* getNativeThreadId(context.providerThread);
6222
+ const claimed = yield* turnTerminalizationPermit.withPermits(1)(
6223
+ Effect.gen(function* () {
6224
+ if (
6225
+ (yield* Ref.get(activeTurns)).get(terminal.nativeTurnId) !== context ||
6226
+ goalStops.has(nativeThreadId) ||
6227
+ (yield* Ref.get(interruptingNativeTurns)).has(terminal.nativeTurnId)
6228
+ )
6229
+ return false;
6230
+ capacityContinuations.set(nativeThreadId, { context, retry: state });
6231
+ yield* finalizeCodexTurn({ ...terminal, continueRun: true });
6232
+ return true;
6233
+ }),
6234
+ );
6235
+ if (!claimed) return;
6236
+ // Notifications before the RPC response also take the terminalization permit.
6237
+ yield* startNativeTurn(
6238
+ {
6239
+ ...context.input,
6240
+ providerTurnOrdinal: context.providerTurnOrdinal + 1,
6241
+ },
6242
+ [],
6243
+ ).pipe(
6244
+ Effect.catch((cause) =>
6245
+ Effect.gen(function* () {
6246
+ if (yield* Deferred.isDone(pending)) return;
6247
+ yield* Effect.logWarning(
6248
+ "orchestration-v2.codex-capacity-retry-start-failed",
6249
+ {
6250
+ providerTurnId: context.providerTurnId,
6251
+ cause,
6252
+ },
6253
+ );
6254
+ yield* emitProviderEvent({
6255
+ type: "turn_item.updated",
6256
+ driver: CODEX_PROVIDER,
6257
+ turnItem: codexRetryTurnItem(
6258
+ context,
6259
+ state,
6260
+ "failed",
6261
+ yield* DateTime.now,
6262
+ ),
6263
+ });
6264
+ yield* Deferred.succeed(pending, undefined);
6265
+ yield* turnTerminalizationPermit.withPermits(1)(
6266
+ emitOrDeferRootTerminal({ ...terminal, providerRetry: state }),
6267
+ );
6268
+ }),
6269
+ ),
6270
+ Effect.ensuring(
6271
+ Effect.sync(() => {
6272
+ if (capacityContinuations.get(nativeThreadId)?.context === context)
6273
+ capacityContinuations.delete(nativeThreadId);
6274
+ }),
6275
+ ),
6276
+ );
6277
+ }),
6278
+ ),
6279
+ Effect.forkIn(scope, { startImmediately: true }),
6280
+ );
6281
+ return true;
6282
+ }),
6283
+ );
6284
+ });
6285
+
6286
yield* client.handleServerNotification("turn/completed", (payload) =>
6287
Effect.gen(function* () {
6288
const context = (yield* Ref.get(activeTurns)).get(payload.turn.id);
@@ -6104,7 +6295,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
6295
(yield* Ref.get(interruptingNativeTurns)).has(payload.turn.id)
6296
? "interrupted"
6297
: nativeStatus;
6107
- yield* finalizeCodexTurn({
6298
+ const terminal = {
6299
context,
6300
nativeTurnId: payload.turn.id,
6301
status,
@@ -6117,7 +6308,8 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
6308
? {}
6309
: { failureErrorInfo: payload.turn.error.codexErrorInfo }),
6310
}),
6120
- });
6311
+ };
6312
+ if (!(yield* retryCapacityFailure(terminal))) yield* finalizeCodexTurn(terminal);
6313
}),
6314
);
6315
@@ -6231,10 +6423,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
6423
goalStops.set(goalThreadId, (goalStops.get(goalThreadId) ?? 0) + 1);
6424
return yield* Effect.gen(function* () {
6425
// The run can settle while the pause is in flight, which drops its record.
6234
- const requestedTurnId = latestGoalTurnId(turnInput.providerTurnId);
6426
+ const requestedTurnId = latestRunTurnId(turnInput.providerTurnId);
6427
const heldBefore = goalHolds.get(goalThreadId)?.context;
6428
const turnsBefore = [
6237
- ...(goalRuns.get(requestedTurnId) ?? []),
6429
+ ...(continuedRuns.get(requestedTurnId) ?? []),
6430
...Array.from((yield* Ref.get(activeTurns)).values()).filter(
6431
(context) => context.providerTurnId === requestedTurnId,
6432
),
@@ -6258,13 +6450,13 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
6450
}
6451
return yield* turnTerminalizationPermit.withPermits(1)(
6452
Effect.gen(function* () {
6261
- const providerTurnId = latestGoalTurnId(turnInput.providerTurnId);
6453
+ const providerTurnId = latestRunTurnId(turnInput.providerTurnId);
6454
const hold = goalHolds.get(goalThreadId);
6455
const held = hold !== undefined && hold.context.providerTurnId === providerTurnId;
6456
const goalTurns: ReadonlyArray<ActiveCodexTurnContext> = Array.from(
6457
new Set([
6458
...turnsBefore,
6267
- ...(goalRuns.get(providerTurnId) ?? []),
6459
+ ...(continuedRuns.get(providerTurnId) ?? []),
6460
...(held ? [hold.context] : []),
6461
]),
6462
);
@@ -6319,6 +6511,8 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
6511
return next;
6512
});
6513
const started = yield* client.request("turn/start", turnStartParams);
6514
+ // A turn/started notification may have registered and completed this turn already.
6515
+ if ((yield* Ref.get(pendingRootTurns)).get(threadId) !== turnInput) return;
6516
yield* registerRootTurn({
6517
turnInput,
6518
nativeTurnId: started.turn.id,
@@ -6331,6 +6525,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
6525
Effect.ensuring(
6526
Effect.flatMap(getNativeThreadId(turnInput.providerThread), (threadId) =>
6527
Ref.update(pendingRootTurns, (current) => {
6528
+ if (current.get(threadId) !== turnInput) return current;
6529
const updated = new Map(current);
6530
updated.delete(threadId);
6531
return updated;
@@ -6764,15 +6959,22 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
6959
steerTurn: Effect.fn("CodexAdapterV2.steerTurn")(
6960
function* (turnInput) {
6961
const threadId = yield* getNativeThreadId(turnInput.providerThread);
6767
- const latestTurnId = latestGoalTurnId(turnInput.providerTurnId);
6962
+ const latestTurnId = latestRunTurnId(turnInput.providerTurnId);
6963
const held = heldGoalTurn(turnInput.providerThread, latestTurnId);
6964
const providerTurnId =
6965
held === undefined
6966
? latestTurnId
6967
: ((yield* Deferred.await(held.hold.next))?.providerTurnId ?? latestTurnId);
6773
- const activeTurn = Array.from((yield* Ref.get(activeTurns)).values()).find(
6968
+ let activeTurn = Array.from((yield* Ref.get(activeTurns)).values()).find(
6969
(candidate) => candidate.providerTurnId === providerTurnId,
6970
);
6971
+ const startingRetry = capacityContinuations.get(threadId)?.context;
6972
+ if (activeTurn === undefined && startingRetry?.providerTurnId === providerTurnId) {
6973
+ activeTurn = startingRetry;
6974
+ }
6975
+ while (activeTurn?.capacityRetryPending !== undefined) {
6976
+ activeTurn = yield* Deferred.await(activeTurn.capacityRetryPending);
6977
+ }
6978
if (activeTurn === undefined) {
6979
return yield* toProtocolError(
6980
`Provider turn ${turnInput.providerTurnId} is not active and cannot be steered.`,
@@ -6822,6 +7024,24 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
7024
),
7025
interruptTurn: Effect.fn("CodexAdapterV2.interruptTurn")(
7026
function* (requestedInput) {
7027
+ const nativeThreadId = requestedInput.providerThread.nativeThreadRef?.nativeId;
7028
+ const capacityRetry =
7029
+ nativeThreadId == null
7030
+ ? undefined
7031
+ : capacityContinuations.get(nativeThreadId)?.context;
7032
+ const startingRetry =
7033
+ capacityRetry?.providerTurnId === latestRunTurnId(requestedInput.providerTurnId)
7034
+ ? capacityRetry
7035
+ : undefined;
7036
+ if (startingRetry?.capacityRetryPending !== undefined) {
7037
+ const started = yield* Deferred.await(startingRetry.capacityRetryPending).pipe(
7038
+ Effect.timeoutOption("10 seconds"),
7039
+ );
7040
+ if (Option.isNone(started))
7041
+ return yield* toProtocolError(
7042
+ "Codex did not start the capacity retry within 10 seconds; Stop could not be delivered.",
7043
+ );
7044
+ }
7045
const { turnInput, goalTurns } = yield* resolveGoalStopTarget(requestedInput);
7046
const [activeTurnContexts, settledTurnContexts] =
7047
yield* turnTerminalizationPermit.withPermits(1)(
@@ -6842,7 +7062,11 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
7062
)
7063
: undefined);
7064
const lineageRoots = Array.from(
6845
- new Set([...(activeTurn === undefined ? [] : [activeTurn]), ...goalTurns]),
7065
+ new Set([
7066
+ ...(activeTurn === undefined ? [] : [activeTurn]),
7067
+ ...(startingRetry === undefined ? [] : [startingRetry]),
7068
+ ...goalTurns,
7069
+ ]),
7070
);
7071
const inLineage = (context: ActiveCodexTurnContext) =>
7072
lineageRoots.some(
@@ -7053,6 +7277,15 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
7277
yield* Effect.gen(function* () {
7278
for (const target of interruptTargets) {
7279
const context = target.context;
7280
+ if (context.capacityRetryPending !== undefined) {
7281
+ yield* finalizeCodexTurn({
7282
+ context,
7283
+ nativeTurnId: context.nativeTurnId,
7284
+ status: "interrupted",
7285
+ completedAt: yield* DateTime.now,
7286
+ });
7287
+ continue;
7288
+ }
7289
// Null start timestamps ACK queued turns; interruption must wait for turn/started.
7290
if (context.nativeStartReady !== undefined) {
7291
const ready = yield* Deferred.await(context.nativeStartReady).pipe(
@@ -7193,7 +7426,22 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
7426
);
7427
},
7428
(effect, requestedInput) =>
7196
- effect.pipe(
7429
+ Effect.acquireUseRelease(
7430
+ Effect.sync(() => {
7431
+ const nativeThreadId = requestedInput.providerThread.nativeThreadRef?.nativeId;
7432
+ if (nativeThreadId != null)
7433
+ goalStops.set(nativeThreadId, (goalStops.get(nativeThreadId) ?? 0) + 1);
7434
+ return nativeThreadId;
7435
+ }),
7436
+ () => effect,
7437
+ (nativeThreadId) =>
7438
+ Effect.sync(() => {
7439
+ if (nativeThreadId == null) return;
7440
+ const stops = (goalStops.get(nativeThreadId) ?? 1) - 1;
7441
+ if (stops > 0) goalStops.set(nativeThreadId, stops);
7442
+ else goalStops.delete(nativeThreadId);
7443
+ }),
7444
+ ).pipe(
7445
Effect.mapError(
7446
(cause) =>
7447
new ProviderAdapter.ProviderAdapterInterruptError({
apps/server/src/orchestration-v2/ProviderTurnControlService.ts
+5
-1
@@ -127,7 +127,11 @@ export const layer: Layer.Layer<
127
const interruptProviderThread = targetsRecordedSession
128
? providerThread
129
: { ...providerThread, providerSessionId: input.providerSessionId };
130
- if (providerTurn.status !== "running") {
130
+ const continuingRun =
131
+ input.operation === "steer" &&
132
+ context.run?.status === "running" &&
133
+ context.run.activeAttemptId === providerTurn.runAttemptId;
134
+ if (providerTurn.status !== "running" && !continuingRun) {
135
if (input.operation === "steer") {
136
return yield* new ProviderTurnControlError({
137
threadId: input.threadId,
apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts
+25
-1
@@ -42,6 +42,7 @@ it.effect.each(
42
[true, "without native steering"],
43
[true, "with interrupting native steering"],
44
[true, "settled only"],
45
+ [false, "during continuation"],
46
] as const
47
).map(([mailbox, timing]) => ({
48
mailbox,
@@ -135,7 +136,7 @@ it.effect.each(
136
steerTurn: (turn) =>
137
Effect.gen(function* () {
138
steerCalls += 1;
138
- if (timing === "after delivery") return;
139
+ if (timing === "after delivery" || timing === "during continuation") return;
140
yield* Deferred.succeed(steerEntered, undefined);
141
yield* Deferred.await(rejectSteer);
142
return yield* new ProviderAdapter.ProviderAdapterSteerRunError({
@@ -284,6 +285,29 @@ it.effect.each(
285
: {}),
286
});
287
if (timing !== "before dispatch") yield* dispatchSteer;
288
+ if (timing === "during continuation") {
289
+ const projection = yield* orchestrator.getThreadProjection(threadId);
290
+ const turn = projection.providerTurns[0]!;
291
+ const failed = yield* watch(
292
+ (event) => event.type === "provider-turn.updated" && event.payload.status === "failed",
293
+ );
294
+ yield* Queue.offer(events, {
295
+ type: "provider_turn.updated",
296
+ driver,
297
+ providerTurn: { ...turn, status: "failed", completedAt: yield* DateTime.now },
298
+ });
299
+ yield* Fiber.join(failed);
300
+ yield* worker.drain();
301
+ assert.equal(steerCalls, 1);
302
+ assert.lengthOf(started, 1);
303
+ const continued = yield* orchestrator.getThreadProjection(threadId);
304
+ assert.equal(continued.runs[0]?.status, "running");
305
+ assert.equal(
306
+ continued.messages.find((message) => message.id === messageId)?.runId,
307
+ first.runId,
308
+ );
309
+ return;
310
+ }
311
if (timing === "after delivery") yield* worker.drain();
312
const delivery =
313
timing === "during delivery" ? yield* worker.runOnce.pipe(Effect.forkScoped) : null;