@devries-cameron20 / t3code Mirror · read-only / commits / 740de0e642
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 14:30 UTC

fix(server): retry Codex capacity failures with empty input (#25611)

Co-authored-by: maria-rcks <maria@kuuro.net>

Yash Singh committed Oct 27, 2026 at 14:36 UTC 740de0e642d630a08e37d0780fa99c178c27808e
4 files changed +1124 -69
apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts
+779
@@ -2214,6 +2214,436 @@ describe("CodexAdapterV2 post-settle continuation", () => {
2214 ),
2215 );
2216
2217 + it.effect.each([
2218 + "recover",
2219 + "completion-only",
2220 + "exhaust",
2221 + "stop",
2222 + "stop-continued",
2223 + "stop-starting",
2224 + "started-first",
2225 + "completed-first",
2226 + "new-run",
2227 + "steer",
2228 + "start-fails",
2229 + ] as const)("handles terminal Codex capacity failures when retries %s", (outcome) =>
2230 + Effect.scoped(
2231 + Effect.gen(function* () {
2232 + const scenario = `capacity-${outcome}`;
2233 + const nativeThreadId = `native-${scenario}`;
2234 + const nativeTurnId = (attempt: number) => `${nativeThreadId}-${attempt}`;
2235 + const prompt = "Continue the task.";
2236 + const capacityError = {
2237 + message: "Selected model is at capacity. Please try a different model.",
2238 + codexErrorInfo: "serverOverloaded",
2239 + };
2240 + const failureEntries = (attempt: number): Array<CodexReplay.CodexAppServerReplayEntry> => [
2241 + {
2242 + type: "emit_inbound",
2243 + frame: {
2244 + method: "error",
2245 + params: {
2246 + threadId: nativeThreadId,
2247 + turnId: nativeTurnId(attempt),
2248 + willRetry: false,
2249 + error: capacityError,
2250 + },
2251 + },
2252 + },
2253 + {
2254 + type: "emit_inbound",
2255 + frame: {
2256 + method: "turn/completed",
2257 + params: {
2258 + threadId: nativeThreadId,
2259 + turn: {
2260 + ...makeCodexReplayTurn({ id: nativeTurnId(attempt), status: "failed" }),
2261 + error: capacityError,
2262 + },
2263 + },
2264 + },
2265 + },
2266 + ];
2267 + const entries = codexReplayPreamble({
2268 + nativeThreadId,
2269 + nativeTurnId: nativeTurnId(0),
2270 + prompt,
2271 + });
2272 + entries.push(
2273 + ...(outcome === "completion-only" ? failureEntries(0).slice(1) : failureEntries(0)),
2274 + );
2275 + if (outcome === "completion-only") entries.push(...failureEntries(0).slice(1));
2276 + const retries = outcome === "exhaust" ? 5 : outcome === "stop" ? 0 : 1;
2277 + const steeredText = "Use the smaller change.";
2278 + for (let attempt = 1; attempt <= retries; attempt++) {
2279 + const start = codexReplayPreamble({
2280 + nativeThreadId,
2281 + nativeTurnId: nativeTurnId(attempt),
2282 + prompt,
2283 + }).slice(-3);
2284 + if (
2285 + outcome === "started-first" ||
2286 + outcome === "stop-starting" ||
2287 + outcome === "completed-first" ||
2288 + outcome === "new-run"
2289 + ) {
2290 + start.splice(1, 0, ...start.splice(2, 1));
2291 + }
2292 + for (const entry of start) {
2293 + if (entry.type === "expect_outbound") {
2294 + entries.push({
2295 + ...entry,
2296 + frame: {
2297 + id: 3 + attempt,
2298 + method: "turn/start",
2299 + params: {
2300 + threadId: nativeThreadId,
2301 + input: [],
2302 + cwd: "/workspace",
2303 + model: "gpt-5.4",
2304 + approvalPolicy: "never",
2305 + approvalsReviewer: "user",
2306 + sandboxPolicy: { type: "dangerFullAccess" },
2307 + summary: "detailed",
2308 + collaborationMode: { mode: "default", settings: { model: "gpt-5.4" } },
2309 + },
2310 + },
2311 + });
2312 + } else if (entry.type === "emit_inbound" && entry.label === "turn/start") {
2313 + if (outcome === "completed-first" || outcome === "new-run")
2314 + entries.push({
2315 + type: "emit_inbound",
2316 + frame: {
2317 + method: "turn/completed",
2318 + params: {
2319 + threadId: nativeThreadId,
2320 + turn: makeCodexReplayTurn({ id: nativeTurnId(attempt), status: "completed" }),
2321 + },
2322 + },
2323 + });
2324 + if (outcome === "new-run") {
2325 + entries.push(
2326 + ...codexReplayPreamble({
2327 + nativeThreadId,
2328 + nativeTurnId: nativeTurnId(2),
2329 + prompt: "Next task.",
2330 + })
2331 + .slice(-3)
2332 + .map((next, index) => (index < 2 ? withReplayRequestId(next, 5) : next)),
2333 + ...failureEntries(2),
2334 + );
2335 + }
2336 + entries.push(
2337 + outcome === "start-fails"
2338 + ? {
2339 + type: "emit_inbound",
2340 + frame: {
2341 + id: 3 + attempt,
2342 + error: { code: -32603, message: "Start failed" },
2343 + },
2344 + }
2345 + : withReplayRequestId(entry, 3 + attempt),
2346 + );
2347 + } else if (outcome !== "start-fails") {
2348 + entries.push(
2349 + outcome === "stop-starting" && entry.type === "emit_inbound"
2350 + ? { ...entry, afterMs: 1_000 }
2351 + : entry,
2352 + );
2353 + if (outcome === "stop-starting") {
2354 + entries.push(
2355 + {
2356 + type: "expect_outbound",
2357 + frame: {
2358 + id: 5,
2359 + method: "turn/interrupt",
2360 + params: { threadId: nativeThreadId, turnId: nativeTurnId(attempt) },
2361 + },
2362 + },
2363 + { type: "emit_inbound", frame: { id: 5, result: {} } },
2364 + );
2365 + }
2366 + }
2367 + }
2368 + if (outcome === "exhaust") entries.push(...failureEntries(attempt));
2369 + if (
2370 + outcome === "recover" ||
2371 + outcome === "completion-only" ||
2372 + outcome === "started-first" ||
2373 + outcome === "stop-starting"
2374 + ) {
2375 + entries.push({
2376 + type: "emit_inbound",
2377 + frame: {
2378 + method: "turn/completed",
2379 + params: {
2380 + threadId: nativeThreadId,
2381 + turn: makeCodexReplayTurn({
2382 + id: nativeTurnId(attempt),
2383 + status: outcome === "stop-starting" ? "interrupted" : "completed",
2384 + }),
2385 + },
2386 + },
2387 + });
2388 + }
2389 + if (outcome === "stop-continued" || outcome === "steer") {
2390 + const steering = outcome === "steer";
2391 + entries.push(
2392 + {
2393 + type: "expect_outbound",
2394 + frame: {
2395 + id: 5,
2396 + method: steering ? "turn/steer" : "turn/interrupt",
2397 + params: steering
2398 + ? {
2399 + threadId: nativeThreadId,
2400 + expectedTurnId: nativeTurnId(attempt),
2401 + input: [{ type: "text", text: steeredText }],
2402 + }
2403 + : { threadId: nativeThreadId, turnId: nativeTurnId(attempt) },
2404 + },
2405 + },
2406 + {
2407 + type: "emit_inbound",
2408 + frame: { id: 5, result: steering ? { turnId: nativeTurnId(attempt) } : {} },
2409 + },
2410 + {
2411 + type: "emit_inbound",
2412 + frame: {
2413 + method: "turn/completed",
2414 + params: {
2415 + threadId: nativeThreadId,
2416 + turn: makeCodexReplayTurn({
2417 + id: nativeTurnId(attempt),
2418 + status: steering ? "completed" : "interrupted",
2419 + }),
2420 + },
2421 + },
2422 + },
2423 + );
2424 + }
2425 + }
2426 + const milestones = yield* Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event>();
2427 + const starts: Array<unknown> = [];
2428 + const retryStarted = yield* Deferred.make<void>();
2429 + const harness = yield* makeCodexReplayHarness(
2430 + makeCodexReplayTranscript({ scenario, entries }),
2431 + (event) =>
2432 + event.type === "turn.terminal" ||
2433 + (event.type === "turn_item.updated" &&
2434 + event.turnItem.type === "error" &&
2435 + event.turnItem.status === "running")
2436 + ? Queue.offer(milestones, event)
2437 + : Effect.void,
2438 + (method, params) =>
2439 + Effect.gen(function* () {
2440 + if (method === "turn/start") {
2441 + starts.push(params);
2442 + if (starts.length === 2) yield* Deferred.succeed(retryStarted, undefined);
2443 + }
2444 + }),
2445 + );
2446 + const turnInput = makeCodexTestTurnInput({
2447 + threadId: harness.threadId,
2448 + providerThread: harness.providerThread,
2449 + now: yield* DateTime.now,
2450 + attemptId: RunAttemptId.make(scenario),
2451 + text: prompt,
2452 + });
2453 + yield* harness.runtime.startTurn(turnInput);
2454 + for (let attempt = 1; attempt <= Math.max(retries, 1); attempt++) {
2455 + const milestone = yield* Queue.take(milestones);
2456 + assert.equal(milestone.type, "turn_item.updated");
2457 + if (milestone.type !== "turn_item.updated" || milestone.turnItem.type !== "error") return;
2458 + const providerTurnId = milestone.turnItem.providerTurnId;
2459 + assert.isNotNull(providerTurnId);
2460 + if (providerTurnId === null) return;
2461 + assert.deepEqual(milestone.turnItem.retry, {
2462 + attempt,
2463 + maxAttempts: 5,
2464 + retryDelayMs: Math.min(2_000 * 2 ** (attempt - 1), 30_000),
2465 + });
2466 + assert.isEmpty(harness.terminalEvents());
2467 + if (outcome === "stop") {
2468 + yield* harness.runtime.interruptTurn({
2469 + providerThread: harness.providerThread,
2470 + providerTurnId,
2471 + });
2472 + }
2473 + const steering =
2474 + outcome === "steer"
2475 + ? yield* harness.runtime
2476 + .steerTurn({
2477 + ...turnInput,
2478 + providerTurnId,
2479 + message: { ...turnInput.message, text: steeredText },
2480 + })
2481 + .pipe(Effect.forkScoped)
2482 + : undefined;
2483 + yield* TestClock.adjust(Math.min(2_000 * 2 ** (attempt - 1), 30_000));
2484 + if (outcome === "stop-starting") {
2485 + yield* Deferred.await(retryStarted);
2486 + const stopping = yield* harness.runtime
2487 + .interruptTurn({ providerThread: harness.providerThread, providerTurnId })
2488 + .pipe(Effect.forkScoped);
2489 + yield* TestClock.adjust("1 second");
2490 + yield* Fiber.join(stopping);
2491 + }
2492 + if (steering !== undefined) yield* Fiber.join(steering);
2493 + if (outcome === "stop-continued") {
2494 + yield* harness.until(() =>
2495 + harness.events.some(
2496 + (event) =>
2497 + event.type === "provider_turn.updated" &&
2498 + event.providerTurn.nativeTurnRef?.nativeId === nativeTurnId(1),
2499 + ),
2500 + );
2501 + yield* harness.runtime.interruptTurn({
2502 + providerThread: harness.providerThread,
2503 + providerTurnId,
2504 + });
2505 + }
2506 + }
2507 + yield* harness.awaitTerminal(1);
2508 + if (outcome === "new-run") {
2509 + assert.equal((yield* Queue.take(milestones)).type, "turn.terminal");
2510 + yield* harness.runtime.startTurn({
2511 + ...turnInput,
2512 + attemptId: RunAttemptId.make("capacity-next-task"),
2513 + message: { ...turnInput.message, text: "Next task." },
2514 + });
2515 + const nextRetry = yield* Queue.take(milestones);
2516 + assert.equal(nextRetry.type, "turn_item.updated");
2517 + if (nextRetry.type !== "turn_item.updated" || nextRetry.turnItem.type !== "error") return;
2518 + assert.equal(nextRetry.turnItem.retry?.attempt, 1);
2519 + return;
2520 + }
2521 + const terminal = harness.terminalEvents()[0];
2522 + assert.equal(
2523 + terminal?.status,
2524 + outcome === "recover" ||
2525 + outcome === "completion-only" ||
2526 + outcome === "steer" ||
2527 + outcome === "started-first" ||
2528 + outcome === "completed-first"
2529 + ? "completed"
2530 + : outcome === "stop" || outcome === "stop-continued" || outcome === "stop-starting"
2531 + ? "interrupted"
2532 + : "failed",
2533 + );
2534 + assert.lengthOf(starts, 1 + retries);
2535 + assert.lengthOf(harness.terminalEvents(), 1);
2536 + assert.lengthOf(
2537 + harness.events.filter(
2538 + (event) =>
2539 + event.type === "provider_turn.updated" && event.providerTurn.status === "running",
2540 + ),
2541 + outcome === "start-fails" ? 1 : 1 + retries,
2542 + );
2543 + const retryItems = harness.events.flatMap((event) =>
2544 + event.type === "turn_item.updated" && event.turnItem.type === "error"
2545 + ? [event.turnItem]
2546 + : [],
2547 + );
2548 + assert.isTrue(
2549 + ["completed", "failed", "interrupted"].includes(retryItems.at(-1)?.status ?? ""),
2550 + );
2551 + assert.lengthOf(new Set(retryItems.map((item) => item.id)), 1);
2552 +
2553 + if (outcome === "recover") {
2554 + const fs = yield* FileSystem.FileSystem;
2555 + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-capacity-retry-" });
2556 + const localTranscript = yield* decodeReplayTranscriptJson(
2557 + (yield* encodeReplayTranscriptJson(
2558 + makeCodexReplayTranscript({ scenario, entries }),
2559 + )).replaceAll(yield* encodeStringJson("/workspace"), yield* encodeStringJson(cwd)),
2560 + );
2561 + const driver = yield* CodexReplay.makeReplayDriver(localTranscript);
2562 + yield* Effect.gen(function* () {
2563 + const orchestrator = yield* Orchestrator.OrchestratorV2;
2564 + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
2565 + const threadId = ThreadId.make("thread:capacity-retry");
2566 + yield* orchestrator.dispatch({
2567 + type: "thread.create",
2568 + commandId: CommandId.make("create-capacity-retry"),
2569 + threadId,
2570 + projectId: ProjectId.make("project:capacity-retry"),
2571 + title: "Capacity retry",
2572 + modelSelection: CODEX_TEST_MODEL_SELECTION,
2573 + runtimeMode: "full-access",
2574 + interactionMode: "default",
2575 + branch: null,
2576 + worktreePath: cwd,
2577 + createdBy: "user",
2578 + creationSource: "web",
2579 + });
2580 + const retrying = yield* orchestrator.streamDomainEvents.pipe(
2581 + Stream.filter(
2582 + (event) =>
2583 + event.type === "turn-item.updated" &&
2584 + event.payload.type === "error" &&
2585 + event.payload.retry != null &&
2586 + event.payload.status === "running",
2587 + ),
2588 + Stream.runHead,
2589 + Effect.forkChild({ startImmediately: true }),
2590 + );
2591 + yield* orchestrator.dispatch({
2592 + type: "message.dispatch",
2593 + commandId: CommandId.make("start-capacity-retry"),
2594 + threadId,
2595 + messageId: MessageId.make("message:capacity-retry"),
2596 + text: prompt,
2597 + attachments: [],
2598 + createdBy: "user",
2599 + creationSource: "web",
2600 + dispatchMode: { type: "start_immediately" },
2601 + });
2602 + yield* worker.drain();
2603 + yield* Fiber.join(retrying);
2604 + assert.equal(
2605 + (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)?.status,
2606 + "running",
2607 + );
2608 + const completed = yield* orchestrator.streamDomainEvents.pipe(
2609 + Stream.filter(
2610 + (event) => event.type === "run.updated" && event.payload.status === "waiting",
2611 + ),
2612 + Stream.runHead,
2613 + Effect.forkChild({ startImmediately: true }),
2614 + );
2615 + yield* TestClock.adjust("2 seconds");
2616 + yield* Fiber.join(completed);
2617 + yield* worker.drain();
2618 + const projection = yield* orchestrator.getThreadProjection(threadId);
2619 + assert.equal(projection.runs.at(-1)?.status, "completed");
2620 + assert.deepEqual(
2621 + projection.providerTurns.map((turn) => turn.status),
2622 + ["failed", "completed"],
2623 + );
2624 + assert.lengthOf(
2625 + projection.messages.filter((message) => message.role === "user"),
2626 + 1,
2627 + );
2628 + assert.isNull((yield* Ref.get(driver.state)).failure);
2629 + }).pipe(
2630 + Effect.provide(
2631 + ProviderReplayHarness.layerWithRegistry(
2632 + { name: "codex-capacity-retry", runtimePolicyOverride: { cwd } },
2633 + CodexAdapterV2Testkit.layer({ transcript: localTranscript, driver }),
2634 + { runEffectWorker: false },
2635 + ),
2636 + ),
2637 + );
2638 + }
2639 + }).pipe(
2640 + Effect.provide(
2641 + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer),
2642 + ),
2643 + ),
2644 + ),
2645 + );
2646 +
2647 it.effect("resolves retryable app-server errors on resumed provider activity", () =>
2648 Effect.scoped(
2649 Effect.gen(function* () {
@@ -5007,6 +5437,265 @@ describe("CodexAdapterV2 post-settle continuation", () => {
5437 };
5438 };
5439
5440 + it.effect.each(["completed", "failed", "interrupted", "starting", "replied-last"] as const)(
5441 + "handles a late native child after the capacity retry %s",
5442 + (parentStatus) =>
5443 + Effect.scoped(
5444 + Effect.gen(function* () {
5445 + const retriedTurnId = "native-capacity-recovered-parent";
5446 + const retrying = yield* Deferred.make<void>();
5447 + const entries: Array<CodexReplay.CodexAppServerReplayEntry> = [
5448 + ...lateChildPrefix().spawned,
5449 + {
5450 + type: "emit_inbound",
5451 + frame: {
5452 + method: "turn/completed",
5453 + params: {
5454 + threadId: RESUME_NATIVE_THREAD,
5455 + turn: {
5456 + ...makeCodexReplayTurn({ id: RESUME_NATIVE_TURN, status: "failed" }),
5457 + error: {
5458 + message: "Selected model is at capacity.",
5459 + codexErrorInfo: "serverOverloaded",
5460 + },
5461 + },
5462 + },
5463 + },
5464 + },
5465 + {
5466 + type: "expect_outbound",
5467 + frame: {
5468 + id: 4,
5469 + method: "turn/start",
5470 + params: {
5471 + threadId: RESUME_NATIVE_THREAD,
5472 + input: [],
5473 + cwd: "/workspace",
5474 + model: "gpt-5.4",
5475 + approvalPolicy: "never",
5476 + approvalsReviewer: "user",
5477 + sandboxPolicy: { type: "dangerFullAccess" },
5478 + summary: "detailed",
5479 + collaborationMode: { mode: "default", settings: { model: "gpt-5.4" } },
5480 + },
5481 + },
5482 + },
5483 + ...(parentStatus === "starting" ? [childTurnCompleted(RESUME_CHILD_TURN_1)] : []),
5484 + {
5485 + type: "emit_inbound",
5486 + frame: {
5487 + id: 4,
5488 + result: {
5489 + turn: makeCodexReplayTurn({ id: retriedTurnId, status: "inProgress" }),
5490 + },
5491 + },
5492 + },
5493 + {
5494 + type: "emit_inbound",
5495 + frame: {
5496 + method: "turn/started",
5497 + params: {
5498 + threadId: RESUME_NATIVE_THREAD,
5499 + turn: makeCodexReplayTurn({ id: retriedTurnId, status: "inProgress" }),
5500 + },
5501 + },
5502 + },
5503 + {
5504 + type: "emit_inbound",
5505 + frame: {
5506 + method: "turn/completed",
5507 + params: {
5508 + threadId: RESUME_NATIVE_THREAD,
5509 + turn: {
5510 + ...makeCodexReplayTurn({
5511 + id: retriedTurnId,
5512 + status:
5513 + parentStatus === "starting" || parentStatus === "replied-last"
5514 + ? "completed"
5515 + : parentStatus,
5516 + }),
5517 + error:
5518 + parentStatus === "failed"
5519 + ? { message: "Recovery failed.", codexErrorInfo: "internalServerError" }
5520 + : null,
5521 + },
5522 + },
5523 + },
5524 + },
5525 + childAgentMessage({
5526 + id: "child-after-capacity",
5527 + text: "CAPACITY_CHILD_DONE",
5528 + turnId: RESUME_CHILD_TURN_1,
5529 + completedAtMs: 1782622480000,
5530 + afterMs: 60_000,
5531 + }),
5532 + childTurnCompleted(RESUME_CHILD_TURN_1),
5533 + ];
5534 + if (parentStatus === "replied-last") {
5535 + entries.push(...entries.splice(lateChildPrefix().spawned.length + 2, 1));
5536 + }
5537 + const harness = yield* makeCodexReplayHarness(
5538 + makeCodexReplayTranscript({ scenario: "capacity-late-child", entries }),
5539 + (event) =>
5540 + event.type === "turn_item.updated" &&
5541 + event.turnItem.type === "error" &&
5542 + event.turnItem.status === "running"
5543 + ? Deferred.succeed(retrying, undefined)
5544 + : Effect.void,
5545 + );
5546 + yield* harness.runtime.startTurn(
5547 + makeCodexTestTurnInput({
5548 + threadId: harness.threadId,
5549 + providerThread: harness.providerThread,
5550 + now: yield* DateTime.now,
5551 + attemptId: RunAttemptId.make("capacity-late-child"),
5552 + text: RESUME_PROMPT,
5553 + }),
5554 + );
5555 + yield* Deferred.await(retrying);
5556 + yield* TestClock.adjust("2 seconds");
5557 + if (parentStatus === "interrupted") assert.isEmpty(harness.terminalEvents());
5558 + else {
5559 + yield* harness.awaitTerminal(1);
5560 + const terminal = harness.terminalEvents()[0];
5561 + if (terminal?.status === "failed") assert.isTrue(terminal.childTurnsContinue);
5562 + }
5563 + yield* TestClock.adjust("60 seconds");
5564 + yield* harness.until(
5565 + () => harness.subagentUpdates().at(-1)?.subagent.status === "completed",
5566 + );
5567 + yield* harness.awaitTerminal(1);
5568 + assert.lengthOf(
5569 + harness.continuationRequests,
5570 + parentStatus === "completed" || parentStatus === "replied-last" ? 1 : 0,
5571 + );
5572 + if (parentStatus === "completed" || parentStatus === "replied-last")
5573 + assert.include(harness.continuationRequests[0]?.detail, "CAPACITY_CHILD_DONE");
5574 + }).pipe(
5575 + Effect.provide(
5576 + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer),
5577 + ),
5578 + ),
5579 + ),
5580 + );
5581 +
5582 + it.effect("stops native children when capacity retry startup fails", () =>
5583 + Effect.scoped(
5584 + Effect.gen(function* () {
5585 + const retrying = yield* Deferred.make<void>();
5586 + const starting = yield* Deferred.make<void>();
5587 + const release = yield* Deferred.make<void>();
5588 + const interrupts: unknown[] = [];
5589 + let starts = 0;
5590 + const harness = yield* makeCodexReplayHarness(
5591 + makeCodexReplayTranscript({
5592 + scenario: "capacity-start-fails-stop-child",
5593 + entries: [
5594 + ...lateChildPrefix().spawned,
5595 + {
5596 + type: "emit_inbound",
5597 + frame: {
5598 + method: "turn/completed",
5599 + params: {
5600 + threadId: RESUME_NATIVE_THREAD,
5601 + turn: {
5602 + ...makeCodexReplayTurn({ id: RESUME_NATIVE_TURN, status: "failed" }),
5603 + error: { message: "At capacity", codexErrorInfo: "serverOverloaded" },
5604 + },
5605 + },
5606 + },
5607 + },
5608 + {
5609 + type: "expect_outbound",
5610 + frame: {
5611 + id: 4,
5612 + method: "turn/start",
5613 + params: {
5614 + threadId: RESUME_NATIVE_THREAD,
5615 + input: [],
5616 + cwd: "/workspace",
5617 + model: "gpt-5.4",
5618 + approvalPolicy: "never",
5619 + approvalsReviewer: "user",
5620 + sandboxPolicy: { type: "dangerFullAccess" },
5621 + summary: "detailed",
5622 + collaborationMode: { mode: "default", settings: { model: "gpt-5.4" } },
5623 + },
5624 + },
5625 + },
5626 + {
5627 + type: "emit_inbound",
5628 + frame: { id: 4, error: { code: -32603, message: "Start failed" } },
5629 + },
5630 + {
5631 + type: "expect_outbound",
5632 + frame: {
5633 + id: 5,
5634 + method: "turn/interrupt",
5635 + params: { threadId: RESUME_CHILD_THREAD, turnId: RESUME_CHILD_TURN_1 },
5636 + },
5637 + },
5638 + { type: "emit_inbound", frame: { id: 5, result: {} } },
5639 + {
5640 + type: "emit_inbound",
5641 + frame: {
5642 + method: "turn/completed",
5643 + params: {
5644 + threadId: RESUME_CHILD_THREAD,
5645 + turn: makeCodexReplayTurn({ id: RESUME_CHILD_TURN_1, status: "interrupted" }),
5646 + },
5647 + },
5648 + },
5649 + ],
5650 + }),
5651 + (event) =>
5652 + event.type === "turn_item.updated" &&
5653 + event.turnItem.type === "error" &&
5654 + event.turnItem.status === "running"
5655 + ? Deferred.succeed(retrying, undefined)
5656 + : Effect.void,
5657 + (method, params) =>
5658 + Effect.gen(function* () {
5659 + if (method === "turn/interrupt") interrupts.push(params);
5660 + if (method === "turn/start" && ++starts === 2) {
5661 + yield* Deferred.succeed(starting, undefined);
5662 + yield* Deferred.await(release);
5663 + }
5664 + }),
5665 + );
5666 + yield* harness.runtime.startTurn(
5667 + makeCodexTestTurnInput({
5668 + threadId: harness.threadId,
5669 + providerThread: harness.providerThread,
5670 + now: yield* DateTime.now,
5671 + attemptId: RunAttemptId.make("capacity-start-fails-stop-child"),
5672 + text: RESUME_PROMPT,
5673 + }),
5674 + );
5675 + yield* Deferred.await(retrying);
5676 + const root = harness.events.find((event) => event.type === "provider_turn.updated");
5677 + assert.equal(root?.type, "provider_turn.updated");
5678 + if (root?.type !== "provider_turn.updated") return;
5679 + yield* TestClock.adjust("2 seconds");
5680 + yield* Deferred.await(starting);
5681 + const stopping = yield* harness.runtime
5682 + .interruptTurn({
5683 + providerThread: harness.providerThread,
5684 + providerTurnId: root.providerTurn.id,
5685 + })
5686 + .pipe(Effect.forkChild({ startImmediately: true }));
5687 + yield* Deferred.succeed(release, undefined);
5688 + yield* Fiber.join(stopping);
5689 + assert.lengthOf(interrupts, 1);
5690 + assert.equal(harness.subagentUpdates().at(-1)?.subagent.status, "interrupted");
5691 + }).pipe(
5692 + Effect.provide(
5693 + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer),
5694 + ),
5695 + ),
5696 + ),
5697 + );
5698 +
5699 it.effect("names a late child spawned without a prompt in its wake", () =>
5700 Effect.scoped(
5701 Effect.gen(function* () {
@@ -6434,6 +7123,96 @@ describe("CodexAdapterV2 post-settle continuation", () => {
7123 ),
7124 );
7125
7126 + it.effect("Stop prevents a capacity retry while pausing its goal", () =>
7127 + Effect.gen(function* () {
7128 + const nativeTurnId = "goal-capacity-stop";
7129 + const prompt = "Keep going";
7130 + const retrying = yield* Deferred.make<void>();
7131 + const pausing = yield* Deferred.make<void>();
7132 + const requests: Array<string> = [];
7133 + const harness = yield* makeCodexReplayHarness(
7134 + makeCodexReplayTranscript({
7135 + scenario: "goal-capacity-stop",
7136 + entries: [
7137 + ...codexReplayPreamble({ nativeThreadId, nativeTurnId, prompt }),
7138 + notification("goal active", "thread/goal/updated", {
7139 + threadId: nativeThreadId,
7140 + turnId: nativeTurnId,
7141 + goal: codexGoal("active", 10),
7142 + }),
7143 + notification("capacity failure", "turn/completed", {
7144 + threadId: nativeThreadId,
7145 + turn: {
7146 + ...makeCodexReplayTurn({ id: nativeTurnId, status: "failed" }),
7147 + error: {
7148 + message: "Selected model is at capacity.",
7149 + codexErrorInfo: "serverOverloaded",
7150 + },
7151 + },
7152 + }),
7153 + {
7154 + type: "expect_outbound",
7155 + frame: {
7156 + id: 4,
7157 + method: "thread/goal/set",
7158 + params: { threadId: nativeThreadId, status: "paused" },
7159 + },
7160 + },
7161 + {
7162 + type: "emit_inbound",
7163 + afterMs: 3_000,
7164 + frame: { id: 4, result: { goal: codexGoal("paused", 10) } },
7165 + },
7166 + ],
7167 + }),
7168 + (event) =>
7169 + event.type === "turn_item.updated" &&
7170 + event.turnItem.type === "error" &&
7171 + event.turnItem.status === "running"
7172 + ? Deferred.succeed(retrying, undefined)
7173 + : Effect.void,
7174 + (method) =>
7175 + Effect.gen(function* () {
7176 + requests.push(method);
7177 + if (method === "thread/goal/set") yield* Deferred.succeed(pausing, undefined);
7178 + }),
7179 + );
7180 + yield* harness.runtime.startTurn(
7181 + makeCodexTestTurnInput({
7182 + threadId: harness.threadId,
7183 + providerThread: harness.providerThread,
7184 + now: yield* DateTime.now,
7185 + attemptId: RunAttemptId.make("goal-capacity-stop"),
7186 + text: prompt,
7187 + }),
7188 + );
7189 + yield* Deferred.await(retrying);
7190 + const stopping = yield* harness.runtime
7191 + .interruptTurn({
7192 + providerThread: harness.providerThread,
7193 + providerTurnId: (yield* IdAllocator.IdAllocatorV2).derive.providerTurn({
7194 + driver: CodexAdapterV2.CODEX_DRIVER_KIND,
7195 + nativeTurnId,
7196 + }),
7197 + })
7198 + .pipe(Effect.forkScoped);
7199 + yield* Deferred.await(pausing);
7200 + yield* TestClock.adjust("3 seconds");
7201 + assert.lengthOf(
7202 + requests.filter((method) => method === "turn/start"),
7203 + 1,
7204 + );
7205 + yield* Fiber.join(stopping);
7206 + yield* harness.awaitTerminal(1);
7207 + assert.equal(harness.terminalEvents()[0]?.status, "interrupted");
7208 + }).pipe(
7209 + Effect.scoped,
7210 + Effect.provide(
7211 + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer),
7212 + ),
7213 + ),
7214 + );
7215 +
7216 it.effect("pauses an active goal before Stop interrupts its turn", () =>
7217 Effect.gen(function* () {
7218 const prompt = "Keep going";
apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts
+315 -67
@@ -86,6 +86,7 @@ import * as Deferred from "effect/Deferred";
86 import * as Effect from "effect/Effect";
87 import * as Fiber from "effect/Fiber";
88 import * as FileSystem from "effect/FileSystem";
89 +import * as Function from "effect/Function";
90 import * as Layer from "effect/Layer";
91 import * as Option from "effect/Option";
92 import * as Path from "effect/Path";
@@ -152,6 +153,7 @@ import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
153 import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
154
155 const CODEX_PROVIDER = ProviderDriverKind.make("codex");
156 +const CODEX_CAPACITY_MAX_RETRIES = 5;
157 export const CODEX_DRIVER_KIND = CODEX_PROVIDER;
158 export const CODEX_DEFAULT_INSTANCE_ID = defaultInstanceIdForDriver(CODEX_DRIVER_KIND);
159
@@ -1109,6 +1111,8 @@ function withCodexManagedFailure(
1111 }
1112
1113 interface ActiveCodexTurnContext {
1114 + capacityRetryAttempt?: number;
1115 + capacityRetryPending?: Deferred.Deferred<ActiveCodexTurnContext | undefined>;
1116 closedCommandItemIds?: ReadonlySet<string>;
1117 latestProviderFailure?: {
1118 readonly nativeMessage: string;
@@ -1141,6 +1145,7 @@ interface ActiveCodexTurnContext {
1145 }
1146
1147 interface ActiveCodexProviderRetry {
1148 + readonly context?: ActiveCodexTurnContext;
1149 readonly nativeMessage: string;
1150 readonly nativeCode: string | null;
1151 readonly retry: OrchestrationV2ProviderRetry;
@@ -1149,6 +1154,30 @@ interface ActiveCodexProviderRetry {
1154 readonly itemOrdinal: number;
1155 }
1156
1157 +function codexRetryTurnItem(
1158 + activeContext: ActiveCodexTurnContext,
1159 + state: ActiveCodexProviderRetry,
1160 + status: Parameters<typeof makeProviderRetryTurnItem>[0]["status"],
1161 + updatedAt: DateTime.Utc,
1162 +) {
1163 + // A continuation keeps the retry row attached to the native turn that failed.
1164 + const context = state.context ?? activeContext;
1165 + return makeProviderRetryTurnItem({
1166 + driver: CODEX_PROVIDER,
1167 + threadId: context.projectionThreadId,
1168 + runId: context.projectionRunId,
1169 + nodeId: context.providerNodeId,
1170 + providerThreadId: context.providerThread.id,
1171 + providerTurnId: context.providerTurnId,
1172 + itemOrdinal: state.itemOrdinal,
1173 + failure: state.failure,
1174 + retry: state.retry,
1175 + status,
1176 + startedAt: state.startedAt,
1177 + updatedAt,
1178 + });
1179 +}
1180 +
1181 interface TrackedRunningCommandItem {
1182 readonly id: string;
1183 readonly command: string;
@@ -1815,12 +1844,23 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
1844 const goalsByNativeThread = new Map<string, OrchestrationV2ProviderGoal | null>();
1845 const rootProviderThreads = new Map<string, OrchestrationV2ProviderThread>();
1846 const goalHolds = new Map<string, CodexGoalHold>();
1818 - const goalRuns = new Map<ProviderTurnId, Array<ActiveCodexTurnContext>>();
1847 + const continuedRuns = new Map<ProviderTurnId, Array<ActiveCodexTurnContext>>();
1848 + const capacityContinuations = new Map<
1849 + string,
1850 + { readonly context: ActiveCodexTurnContext; readonly retry: ActiveCodexProviderRetry }
1851 + >();
1852 const goalActivations = new Map<string, { stopped: boolean }>();
1853 // Held runs must not complete while Stop resolves its target.
1854 const goalStops = new Map<string, number>();
1822 - const latestGoalTurnId = (providerTurnId: ProviderTurnId) =>
1823 - goalRuns.get(providerTurnId)?.at(-1)?.providerTurnId ?? providerTurnId;
1855 + const latestRunTurnId = (providerTurnId: ProviderTurnId) =>
1856 + continuedRuns.get(providerTurnId)?.at(-1)?.providerTurnId ?? providerTurnId;
1857 + const isContinuedRunDescendant = (
1858 + candidate: ActiveCodexTurnContext,
1859 + context: ActiveCodexTurnContext,
1860 + ) =>
1861 + (continuedRuns.get(context.providerTurnId) ?? [context]).some((ancestor) =>
1862 + isDescendantCodexTurn(candidate, ancestor),
1863 + );
1864
1865 const emitProviderEvent = (event: ProviderAdapter.ProviderAdapterV2Event) =>
1866 Queue.offer(events, event).pipe(Effect.asVoid);
@@ -1882,20 +1922,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
1922 yield* emitProviderEvent({
1923 type: "turn_item.updated",
1924 driver: CODEX_PROVIDER,
1885 - turnItem: makeProviderRetryTurnItem({
1886 - driver: CODEX_PROVIDER,
1887 - threadId: context.projectionThreadId,
1888 - runId: context.projectionRunId,
1889 - nodeId: context.providerNodeId,
1890 - providerThreadId: context.providerThread.id,
1891 - providerTurnId: context.providerTurnId,
1892 - itemOrdinal: providerRetry.itemOrdinal,
1893 - failure: providerRetry.failure,
1894 - retry: providerRetry.retry,
1895 - status: "completed",
1896 - startedAt: providerRetry.startedAt,
1897 - updatedAt,
1898 - }),
1925 + turnItem: codexRetryTurnItem(context, providerRetry, "completed", updatedAt),
1926 });
1927 });
1928
@@ -1934,6 +1961,24 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
1961 fileChangeKindsByItem: new Map(),
1962 runningCompactionItemIds: new Set(),
1963 };
1964 + const nativeThreadId = input.turnInput.providerThread.nativeThreadRef?.nativeId;
1965 + const continuation =
1966 + nativeThreadId == null ? undefined : capacityContinuations.get(nativeThreadId);
1967 + if (continuation !== undefined) {
1968 + context.capacityRetryAttempt = continuation.context.capacityRetryAttempt ?? 0;
1969 + yield* Ref.update(providerRetries, (current) =>
1970 + new Map(current).set(providerTurnId, continuation.retry),
1971 + );
1972 + const turns = continuedRuns.get(continuation.context.providerTurnId) ?? [
1973 + continuation.context,
1974 + ];
1975 + turns.push(context);
1976 + continuedRuns.set(continuation.context.providerTurnId, turns);
1977 + continuedRuns.set(providerTurnId, turns);
1978 + if (continuation.context.capacityRetryPending !== undefined) {
1979 + yield* Deferred.succeed(continuation.context.capacityRetryPending, context);
1980 + }
1981 + }
1982 yield* Ref.update(limitedTurnItems, (current) => {
1983 const next = new Map(current);
1984 next.delete(context.providerThread.id);
@@ -1946,6 +1991,9 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
1991 updated.set(input.nativeTurnId, context);
1992 return updated;
1993 });
1994 + if (continuation !== undefined && nativeThreadId != null) {
1995 + capacityContinuations.delete(nativeThreadId);
1996 + }
1997 yield* emitProviderEvent({
1998 type: "provider_turn.updated",
1999 driver: CODEX_PROVIDER,
@@ -4380,10 +4428,12 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
4428 nativeTurnId: payload.turn.id,
4429 startedAt: codexTimestamp(payload.turn.startedAt),
4430 });
4383 - const goalRun = goalRuns.get(goalHold.context.providerTurnId) ?? [goalHold.context];
4431 + const goalRun = continuedRuns.get(goalHold.context.providerTurnId) ?? [
4432 + goalHold.context,
4433 + ];
4434 goalRun.push(next);
4385 - goalRuns.set(goalHold.context.providerTurnId, goalRun);
4386 - goalRuns.set(next.providerTurnId, goalRun);
4435 + continuedRuns.set(goalHold.context.providerTurnId, goalRun);
4436 + continuedRuns.set(next.providerTurnId, goalRun);
4437 yield* Deferred.succeed(goalHold.next, next);
4438 return;
4439 }
@@ -4493,6 +4543,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
4543 previous?.itemOrdinal ??
4544 (yield* resolveItemOrdinal(context, `terminal-failure:${context.providerTurnId}`));
4545 const state: ActiveCodexProviderRetry = {
4546 + ...(previous?.context === undefined ? {} : { context: previous.context }),
4547 nativeMessage: payload.error.message,
4548 nativeCode: code,
4549 retry,
@@ -4508,20 +4559,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
4559 yield* emitProviderEvent({
4560 type: "turn_item.updated",
4561 driver: CODEX_PROVIDER,
4511 - turnItem: makeProviderRetryTurnItem({
4512 - driver: CODEX_PROVIDER,
4513 - threadId: context.projectionThreadId,
4514 - runId: context.projectionRunId,
4515 - nodeId: context.providerNodeId,
4516 - providerThreadId: context.providerThread.id,
4517 - providerTurnId: context.providerTurnId,
4518 - itemOrdinal,
4519 - failure,
4520 - retry,
4521 - status: "running",
4522 - startedAt: state.startedAt,
4523 - updatedAt,
4524 - }),
4562 + turnItem: codexRetryTurnItem(context, state, "running", updatedAt),
4563 });
4564 }).pipe(Effect.orDie),
4565 );
@@ -5617,8 +5655,16 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
5655 }
5656 : event;
5657 yield* emitProviderEvent(current);
5620 - for (const turn of goalRuns.get(context.providerTurnId) ?? []) {
5621 - goalRuns.delete(turn.providerTurnId);
5658 + const continued = continuedRuns.get(context.providerTurnId) ?? [context];
5659 + if (current.status !== "completed") {
5660 + yield* Ref.update(
5661 + terminalizedNonCompletedNativeTurns,
5662 + (nativeTurns) =>
5663 + new Set([...nativeTurns, ...continued.map((turn) => turn.nativeTurnId)]),
5664 + );
5665 + }
5666 + for (const turn of continued) {
5667 + continuedRuns.delete(turn.providerTurnId);
5668 }
5669 const nativeThreadId = context.providerThread.nativeThreadRef?.nativeId;
5670 const rootProviderThread =
@@ -5656,7 +5702,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
5702 }) {
5703 const event = yield* makeRootTerminalEvent(input);
5704 const hasActiveDescendants = Array.from((yield* Ref.get(activeTurns)).values()).some(
5659 - (candidate) => isDescendantCodexTurn(candidate, input.context),
5705 + (candidate) => isContinuedRunDescendant(candidate, input.context),
5706 );
5707 // Failed roots leave native children working and must settle without them.
5708 // Interrupted roots wait for descendants because Stop interrupts the whole lineage.
@@ -5781,7 +5827,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
5827 for (const [nativeTurnId, deferred] of current) {
5828 if (
5829 !activeTurnContexts.some((candidate) =>
5784 - isDescendantCodexTurn(candidate, deferred.context),
5830 + isContinuedRunDescendant(candidate, deferred.context),
5831 )
5832 ) {
5833 updated.delete(nativeTurnId);
@@ -5803,8 +5849,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
5849 readonly completedAt: DateTime.Utc;
5850 readonly failureMessage?: string;
5851 readonly failureErrorInfo?: CodexErrorInfo;
5852 + readonly continueRun?: boolean;
5853 }) =>
5807 - turnTerminalizationPermit.withPermits(1)(
5854 + // Capacity continuation already holds the permit across finalization and restart.
5855 + (input.continueRun ? Function.identity : turnTerminalizationPermit.withPermits(1))(
5856 Effect.gen(function* () {
5857 const current = (yield* Ref.get(activeTurns)).get(input.nativeTurnId);
5858 if (current !== input.context) {
@@ -5822,31 +5870,33 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
5870 });
5871 if (
5872 providerRetry !== undefined &&
5825 - (input.status !== "failed" || input.context.subagent !== null)
5873 + !input.continueRun &&
5874 + (providerRetry.context !== undefined ||
5875 + input.status !== "failed" ||
5876 + input.context.subagent !== null)
5877 ) {
5878 yield* emitProviderEvent({
5879 type: "turn_item.updated",
5880 driver: CODEX_PROVIDER,
5830 - turnItem: makeProviderRetryTurnItem({
5831 - driver: CODEX_PROVIDER,
5832 - threadId: input.context.projectionThreadId,
5833 - runId: input.context.projectionRunId,
5834 - nodeId: input.context.providerNodeId,
5835 - providerThreadId: input.context.providerThread.id,
5836 - providerTurnId: input.context.providerTurnId,
5837 - itemOrdinal: providerRetry.itemOrdinal,
5838 - failure: providerRetry.failure,
5839 - retry: providerRetry.retry,
5840 - status: providerTurnStatusToTerminal(input.status),
5841 - startedAt: providerRetry.startedAt,
5842 - updatedAt: input.completedAt,
5843 - }),
5881 + turnItem: codexRetryTurnItem(
5882 + input.context,
5883 + providerRetry,
5884 + input.status === "failed" &&
5885 + providerRetry.context !== input.context &&
5886 + providerRetry.context !== undefined
5887 + ? "interrupted"
5888 + : providerTurnStatusToTerminal(input.status),
5889 + input.completedAt,
5890 + ),
5891 });
5892 }
5846 - if (input.status !== "completed") {
5893 + if (input.status !== "completed" && !input.continueRun) {
5894 yield* Ref.update(terminalizedNonCompletedNativeTurns, (current) => {
5895 const updated = new Set(current);
5849 - updated.add(input.nativeTurnId);
5896 + for (const turn of continuedRuns.get(input.context.providerTurnId) ?? [
5897 + input.context,
5898 + ])
5899 + updated.add(turn.nativeTurnId);
5900 return updated;
5901 });
5902 }
@@ -5967,11 +6017,14 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
6017 // Only direct children wake an idle root turn; grandchildren report to their
6018 // own parent, and running root turns receive results natively.
6019 const parent = input.context.subagent.parentContext;
6020 + const parentNativeThreadId = parent.providerThread.nativeThreadRef?.nativeId;
6021 if (
6022 continuationRequests !== undefined &&
6023 parent.subagent === null &&
6024 (input.status === "completed" || input.status === "failed") &&
6025 !(yield* Ref.get(terminalizedNonCompletedNativeTurns)).has(parent.nativeTurnId) &&
6026 + (parentNativeThreadId == null ||
6027 + !capacityContinuations.has(parentNativeThreadId)) &&
6028 ![...(yield* Ref.get(activeTurns)).values()].some(
6029 (active) =>
6030 active.subagent === null &&
@@ -6017,7 +6070,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
6070 input.completedAt,
6071 input.status === "interrupted" || input.status === "failed",
6072 );
6020 - if (input.context.subagent === null) {
6073 + if (input.context.subagent === null && !input.continueRun) {
6074 yield* emitOrDeferRootTerminal({
6075 ...input,
6076 ...(providerRetry === undefined ? {} : { providerRetry }),
@@ -6051,6 +6104,9 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
6104 if (input.context.nativeStartReady !== undefined) {
6105 yield* Deferred.succeed(input.context.nativeStartReady, undefined);
6106 }
6107 + if (!input.continueRun && input.context.capacityRetryPending !== undefined) {
6108 + yield* Deferred.succeed(input.context.capacityRetryPending, undefined);
6109 + }
6110 yield* flushReadyRootTerminals();
6111 if (!retainSettledContext && !interruptInProgress) {
6112 yield* Ref.update(runningCommandItemsByTurn, (current) => {
@@ -6092,6 +6148,141 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
6148 }),
6149 );
6150
6151 + // Codex can end an overloaded turn without retry advice. Continue its
6152 + // persisted history with empty input instead of resending the user message.
6153 + const retryCapacityFailure = Effect.fn("CodexAdapterV2.retryCapacityFailure")(function* (
6154 + terminal: Parameters<typeof finalizeCodexTurn>[0],
6155 + ) {
6156 + const context = terminal.context;
6157 + const code =
6158 + terminal.failureErrorInfo == null
6159 + ? context.latestProviderFailure?.nativeCode
6160 + : codexErrorInfoCode(terminal.failureErrorInfo);
6161 + if (
6162 + terminal.status !== "failed" ||
6163 + code !== "serverOverloaded" ||
6164 + context.subagent !== null ||
6165 + (context.input.message.attachments.length === 0 &&
6166 + context.input.message.text.trim().toLowerCase() === "/compact")
6167 + )
6168 + return false;
6169 +
6170 + return yield* turnTerminalizationPermit.withPermits(1)(
6171 + Effect.gen(function* () {
6172 + if ((yield* Ref.get(activeTurns)).get(terminal.nativeTurnId) !== context) return true;
6173 + if (context.capacityRetryPending !== undefined) return true;
6174 + if (
6175 + (context.capacityRetryAttempt ?? 0) >= CODEX_CAPACITY_MAX_RETRIES ||
6176 + (yield* Ref.get(interruptingNativeTurns)).has(terminal.nativeTurnId)
6177 + )
6178 + return false;
6179 +
6180 + const attempt = (context.capacityRetryAttempt ?? 0) + 1;
6181 + context.capacityRetryAttempt = attempt;
6182 + const pending = yield* Deferred.make<ActiveCodexTurnContext | undefined>();
6183 + context.capacityRetryPending = pending;
6184 + const retryDelayMs = Math.min(2_000 * 2 ** (attempt - 1), 30_000);
6185 + const now = yield* DateTime.now;
6186 + const previous = (yield* Ref.get(providerRetries)).get(context.providerTurnId);
6187 + const state: ActiveCodexProviderRetry = {
6188 + context: previous?.context ?? context,
6189 + nativeMessage:
6190 + terminal.failureMessage ??
6191 + context.latestProviderFailure?.nativeMessage ??
6192 + "Selected model is at capacity.",
6193 + nativeCode: code,
6194 + failure: makeProviderFailure({
6195 + message:
6196 + context.latestProviderFailure?.failure.message ?? terminal.failureMessage,
6197 + code,
6198 + class: "provider_error",
6199 + retryable: true,
6200 + }),
6201 + retry: { attempt, maxAttempts: CODEX_CAPACITY_MAX_RETRIES, retryDelayMs },
6202 + itemOrdinal:
6203 + previous?.itemOrdinal ??
6204 + (yield* resolveItemOrdinal(
6205 + context,
6206 + `terminal-failure:${context.providerTurnId}`,
6207 + )),
6208 + startedAt: previous?.startedAt ?? now,
6209 + };
6210 + yield* Ref.update(providerRetries, (current) =>
6211 + new Map(current).set(context.providerTurnId, state),
6212 + );
6213 + yield* emitProviderEvent({
6214 + type: "turn_item.updated",
6215 + driver: CODEX_PROVIDER,
6216 + turnItem: codexRetryTurnItem(context, state, "running", now),
6217 + });
6218 + yield* Effect.sleep(retryDelayMs).pipe(
6219 + Effect.andThen(
6220 + Effect.gen(function* () {
6221 + const nativeThreadId = yield* getNativeThreadId(context.providerThread);
6222 + const claimed = yield* turnTerminalizationPermit.withPermits(1)(
6223 + Effect.gen(function* () {
6224 + if (
6225 + (yield* Ref.get(activeTurns)).get(terminal.nativeTurnId) !== context ||
6226 + goalStops.has(nativeThreadId) ||
6227 + (yield* Ref.get(interruptingNativeTurns)).has(terminal.nativeTurnId)
6228 + )
6229 + return false;
6230 + capacityContinuations.set(nativeThreadId, { context, retry: state });
6231 + yield* finalizeCodexTurn({ ...terminal, continueRun: true });
6232 + return true;
6233 + }),
6234 + );
6235 + if (!claimed) return;
6236 + // Notifications before the RPC response also take the terminalization permit.
6237 + yield* startNativeTurn(
6238 + {
6239 + ...context.input,
6240 + providerTurnOrdinal: context.providerTurnOrdinal + 1,
6241 + },
6242 + [],
6243 + ).pipe(
6244 + Effect.catch((cause) =>
6245 + Effect.gen(function* () {
6246 + if (yield* Deferred.isDone(pending)) return;
6247 + yield* Effect.logWarning(
6248 + "orchestration-v2.codex-capacity-retry-start-failed",
6249 + {
6250 + providerTurnId: context.providerTurnId,
6251 + cause,
6252 + },
6253 + );
6254 + yield* emitProviderEvent({
6255 + type: "turn_item.updated",
6256 + driver: CODEX_PROVIDER,
6257 + turnItem: codexRetryTurnItem(
6258 + context,
6259 + state,
6260 + "failed",
6261 + yield* DateTime.now,
6262 + ),
6263 + });
6264 + yield* Deferred.succeed(pending, undefined);
6265 + yield* turnTerminalizationPermit.withPermits(1)(
6266 + emitOrDeferRootTerminal({ ...terminal, providerRetry: state }),
6267 + );
6268 + }),
6269 + ),
6270 + Effect.ensuring(
6271 + Effect.sync(() => {
6272 + if (capacityContinuations.get(nativeThreadId)?.context === context)
6273 + capacityContinuations.delete(nativeThreadId);
6274 + }),
6275 + ),
6276 + );
6277 + }),
6278 + ),
6279 + Effect.forkIn(scope, { startImmediately: true }),
6280 + );
6281 + return true;
6282 + }),
6283 + );
6284 + });
6285 +
6286 yield* client.handleServerNotification("turn/completed", (payload) =>
6287 Effect.gen(function* () {
6288 const context = (yield* Ref.get(activeTurns)).get(payload.turn.id);
@@ -6104,7 +6295,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
6295 (yield* Ref.get(interruptingNativeTurns)).has(payload.turn.id)
6296 ? "interrupted"
6297 : nativeStatus;
6107 - yield* finalizeCodexTurn({
6298 + const terminal = {
6299 context,
6300 nativeTurnId: payload.turn.id,
6301 status,
@@ -6117,7 +6308,8 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
6308 ? {}
6309 : { failureErrorInfo: payload.turn.error.codexErrorInfo }),
6310 }),
6120 - });
6311 + };
6312 + if (!(yield* retryCapacityFailure(terminal))) yield* finalizeCodexTurn(terminal);
6313 }),
6314 );
6315
@@ -6231,10 +6423,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
6423 goalStops.set(goalThreadId, (goalStops.get(goalThreadId) ?? 0) + 1);
6424 return yield* Effect.gen(function* () {
6425 // The run can settle while the pause is in flight, which drops its record.
6234 - const requestedTurnId = latestGoalTurnId(turnInput.providerTurnId);
6426 + const requestedTurnId = latestRunTurnId(turnInput.providerTurnId);
6427 const heldBefore = goalHolds.get(goalThreadId)?.context;
6428 const turnsBefore = [
6237 - ...(goalRuns.get(requestedTurnId) ?? []),
6429 + ...(continuedRuns.get(requestedTurnId) ?? []),
6430 ...Array.from((yield* Ref.get(activeTurns)).values()).filter(
6431 (context) => context.providerTurnId === requestedTurnId,
6432 ),
@@ -6258,13 +6450,13 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
6450 }
6451 return yield* turnTerminalizationPermit.withPermits(1)(
6452 Effect.gen(function* () {
6261 - const providerTurnId = latestGoalTurnId(turnInput.providerTurnId);
6453 + const providerTurnId = latestRunTurnId(turnInput.providerTurnId);
6454 const hold = goalHolds.get(goalThreadId);
6455 const held = hold !== undefined && hold.context.providerTurnId === providerTurnId;
6456 const goalTurns: ReadonlyArray<ActiveCodexTurnContext> = Array.from(
6457 new Set([
6458 ...turnsBefore,
6267 - ...(goalRuns.get(providerTurnId) ?? []),
6459 + ...(continuedRuns.get(providerTurnId) ?? []),
6460 ...(held ? [hold.context] : []),
6461 ]),
6462 );
@@ -6319,6 +6511,8 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
6511 return next;
6512 });
6513 const started = yield* client.request("turn/start", turnStartParams);
6514 + // A turn/started notification may have registered and completed this turn already.
6515 + if ((yield* Ref.get(pendingRootTurns)).get(threadId) !== turnInput) return;
6516 yield* registerRootTurn({
6517 turnInput,
6518 nativeTurnId: started.turn.id,
@@ -6331,6 +6525,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
6525 Effect.ensuring(
6526 Effect.flatMap(getNativeThreadId(turnInput.providerThread), (threadId) =>
6527 Ref.update(pendingRootTurns, (current) => {
6528 + if (current.get(threadId) !== turnInput) return current;
6529 const updated = new Map(current);
6530 updated.delete(threadId);
6531 return updated;
@@ -6764,15 +6959,22 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
6959 steerTurn: Effect.fn("CodexAdapterV2.steerTurn")(
6960 function* (turnInput) {
6961 const threadId = yield* getNativeThreadId(turnInput.providerThread);
6767 - const latestTurnId = latestGoalTurnId(turnInput.providerTurnId);
6962 + const latestTurnId = latestRunTurnId(turnInput.providerTurnId);
6963 const held = heldGoalTurn(turnInput.providerThread, latestTurnId);
6964 const providerTurnId =
6965 held === undefined
6966 ? latestTurnId
6967 : ((yield* Deferred.await(held.hold.next))?.providerTurnId ?? latestTurnId);
6773 - const activeTurn = Array.from((yield* Ref.get(activeTurns)).values()).find(
6968 + let activeTurn = Array.from((yield* Ref.get(activeTurns)).values()).find(
6969 (candidate) => candidate.providerTurnId === providerTurnId,
6970 );
6971 + const startingRetry = capacityContinuations.get(threadId)?.context;
6972 + if (activeTurn === undefined && startingRetry?.providerTurnId === providerTurnId) {
6973 + activeTurn = startingRetry;
6974 + }
6975 + while (activeTurn?.capacityRetryPending !== undefined) {
6976 + activeTurn = yield* Deferred.await(activeTurn.capacityRetryPending);
6977 + }
6978 if (activeTurn === undefined) {
6979 return yield* toProtocolError(
6980 `Provider turn ${turnInput.providerTurnId} is not active and cannot be steered.`,
@@ -6822,6 +7024,24 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
7024 ),
7025 interruptTurn: Effect.fn("CodexAdapterV2.interruptTurn")(
7026 function* (requestedInput) {
7027 + const nativeThreadId = requestedInput.providerThread.nativeThreadRef?.nativeId;
7028 + const capacityRetry =
7029 + nativeThreadId == null
7030 + ? undefined
7031 + : capacityContinuations.get(nativeThreadId)?.context;
7032 + const startingRetry =
7033 + capacityRetry?.providerTurnId === latestRunTurnId(requestedInput.providerTurnId)
7034 + ? capacityRetry
7035 + : undefined;
7036 + if (startingRetry?.capacityRetryPending !== undefined) {
7037 + const started = yield* Deferred.await(startingRetry.capacityRetryPending).pipe(
7038 + Effect.timeoutOption("10 seconds"),
7039 + );
7040 + if (Option.isNone(started))
7041 + return yield* toProtocolError(
7042 + "Codex did not start the capacity retry within 10 seconds; Stop could not be delivered.",
7043 + );
7044 + }
7045 const { turnInput, goalTurns } = yield* resolveGoalStopTarget(requestedInput);
7046 const [activeTurnContexts, settledTurnContexts] =
7047 yield* turnTerminalizationPermit.withPermits(1)(
@@ -6842,7 +7062,11 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
7062 )
7063 : undefined);
7064 const lineageRoots = Array.from(
6845 - new Set([...(activeTurn === undefined ? [] : [activeTurn]), ...goalTurns]),
7065 + new Set([
7066 + ...(activeTurn === undefined ? [] : [activeTurn]),
7067 + ...(startingRetry === undefined ? [] : [startingRetry]),
7068 + ...goalTurns,
7069 + ]),
7070 );
7071 const inLineage = (context: ActiveCodexTurnContext) =>
7072 lineageRoots.some(
@@ -7053,6 +7277,15 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
7277 yield* Effect.gen(function* () {
7278 for (const target of interruptTargets) {
7279 const context = target.context;
7280 + if (context.capacityRetryPending !== undefined) {
7281 + yield* finalizeCodexTurn({
7282 + context,
7283 + nativeTurnId: context.nativeTurnId,
7284 + status: "interrupted",
7285 + completedAt: yield* DateTime.now,
7286 + });
7287 + continue;
7288 + }
7289 // Null start timestamps ACK queued turns; interruption must wait for turn/started.
7290 if (context.nativeStartReady !== undefined) {
7291 const ready = yield* Deferred.await(context.nativeStartReady).pipe(
@@ -7193,7 +7426,22 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
7426 );
7427 },
7428 (effect, requestedInput) =>
7196 - effect.pipe(
7429 + Effect.acquireUseRelease(
7430 + Effect.sync(() => {
7431 + const nativeThreadId = requestedInput.providerThread.nativeThreadRef?.nativeId;
7432 + if (nativeThreadId != null)
7433 + goalStops.set(nativeThreadId, (goalStops.get(nativeThreadId) ?? 0) + 1);
7434 + return nativeThreadId;
7435 + }),
7436 + () => effect,
7437 + (nativeThreadId) =>
7438 + Effect.sync(() => {
7439 + if (nativeThreadId == null) return;
7440 + const stops = (goalStops.get(nativeThreadId) ?? 1) - 1;
7441 + if (stops > 0) goalStops.set(nativeThreadId, stops);
7442 + else goalStops.delete(nativeThreadId);
7443 + }),
7444 + ).pipe(
7445 Effect.mapError(
7446 (cause) =>
7447 new ProviderAdapter.ProviderAdapterInterruptError({
apps/server/src/orchestration-v2/ProviderTurnControlService.ts
+5 -1
@@ -127,7 +127,11 @@ export const layer: Layer.Layer<
127 const interruptProviderThread = targetsRecordedSession
128 ? providerThread
129 : { ...providerThread, providerSessionId: input.providerSessionId };
130 - if (providerTurn.status !== "running") {
130 + const continuingRun =
131 + input.operation === "steer" &&
132 + context.run?.status === "running" &&
133 + context.run.activeAttemptId === providerTurn.runAttemptId;
134 + if (providerTurn.status !== "running" && !continuingRun) {
135 if (input.operation === "steer") {
136 return yield* new ProviderTurnControlError({
137 threadId: input.threadId,
apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts
+25 -1
@@ -42,6 +42,7 @@ it.effect.each(
42 [true, "without native steering"],
43 [true, "with interrupting native steering"],
44 [true, "settled only"],
45 + [false, "during continuation"],
46 ] as const
47 ).map(([mailbox, timing]) => ({
48 mailbox,
@@ -135,7 +136,7 @@ it.effect.each(
136 steerTurn: (turn) =>
137 Effect.gen(function* () {
138 steerCalls += 1;
138 - if (timing === "after delivery") return;
139 + if (timing === "after delivery" || timing === "during continuation") return;
140 yield* Deferred.succeed(steerEntered, undefined);
141 yield* Deferred.await(rejectSteer);
142 return yield* new ProviderAdapter.ProviderAdapterSteerRunError({
@@ -284,6 +285,29 @@ it.effect.each(
285 : {}),
286 });
287 if (timing !== "before dispatch") yield* dispatchSteer;
288 + if (timing === "during continuation") {
289 + const projection = yield* orchestrator.getThreadProjection(threadId);
290 + const turn = projection.providerTurns[0]!;
291 + const failed = yield* watch(
292 + (event) => event.type === "provider-turn.updated" && event.payload.status === "failed",
293 + );
294 + yield* Queue.offer(events, {
295 + type: "provider_turn.updated",
296 + driver,
297 + providerTurn: { ...turn, status: "failed", completedAt: yield* DateTime.now },
298 + });
299 + yield* Fiber.join(failed);
300 + yield* worker.drain();
301 + assert.equal(steerCalls, 1);
302 + assert.lengthOf(started, 1);
303 + const continued = yield* orchestrator.getThreadProjection(threadId);
304 + assert.equal(continued.runs[0]?.status, "running");
305 + assert.equal(
306 + continued.messages.find((message) => message.id === messageId)?.runId,
307 + first.runId,
308 + );
309 + return;
310 + }
311 if (timing === "after delivery") yield* worker.drain();
312 const delivery =
313 timing === "during delivery" ? yield* worker.runOnce.pipe(Effect.forkScoped) : null;