@devries-cameron20 / t3code Mirror · read-only / commits / 60f2246358
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 23:15 UTC

fix(server): stopped usage-limit threads no longer auto-resume (#21114)

Co-authored-by: Julius Marminge <julius0216@outlook.com>

fixfon committed Oct 20, 2026 at 08:04 UTC 60f2246358050fa70c7aee3cd685fefaaaaf81db
6 files changed +196 -8
apps/server/src/orchestration-v2/Orchestrator.ts
+71 -1
@@ -27,6 +27,7 @@ import {
27 type ModelSelection,
28 OrchestrationV2Command,
29 type OrchestrationV2InternalCommand,
30 + type OrchestrationV2LimitRecoveryUpdate,
31 type OrchestrationV2ServerCommand,
32 type ThreadPullRequestLink,
33 type ThreadPullRequestWatch,
@@ -312,6 +313,11 @@ export interface OrchestratorV2Shape {
313 readonly dispatch: (
314 command: OrchestrationV2ServerCommand,
315 ) => Effect.Effect<OrchestratorV2DispatchResult, OrchestratorV2Error>;
316 + readonly autoArmLimitRecovery: (input: {
317 + readonly commandId: CommandId;
318 + readonly threadId: ThreadId;
319 + readonly limitRecovery: OrchestrationV2LimitRecoveryUpdate;
320 + }) => Effect.Effect<void, OrchestratorV2Error>;
321 readonly searchThreadStream: (
322 input: OrchestrationV2SearchThreadInput,
323 ) => Stream.Stream<OrchestrationV2SearchThreadResult, OrchestratorV2Error>;
@@ -8978,7 +8984,49 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
8984 readonly now: DateTime.Utc;
8985 }) {
8986 const emitEvent = emit(input.events, input.command);
8981 - const { thread, runs, subagents } = input.projection;
8987 + const { runs, subagents } = input.projection;
8988 + let { thread } = input.projection;
8989 + let recovery = thread.limitRecovery;
8990 + const latest = latestExecutedRun(runs);
8991 + if (latest?.status === "failed") {
8992 + const { turnItems } = yield* projectionStore
8993 + .getThreadRecords(thread.id, ["turnItems"], {
8994 + turnItemTypes: ["error"],
8995 + turnItemRunIds: [latest.id],
8996 + })
8997 + .pipe(mapDispatchError(input.command));
8998 + const failure = latestRootProviderFailure(latest, turnItems);
8999 + if (
9000 + failure?.class === "usage_limit" &&
9001 + failure.resetAt != null &&
9002 + Number.isFinite(Date.parse(failure.resetAt)) &&
9003 + (recovery?.runId !== latest.id || recovery.resetAt !== failure.resetAt)
9004 + ) {
9005 + recovery = {
9006 + runId: latest.id,
9007 + resetAt: failure.resetAt,
9008 + autoResume: false,
9009 + snooze: false,
9010 + };
9011 + }
9012 + }
9013 + if (recovery && (recovery !== thread.limitRecovery || recovery.autoResume)) {
9014 + thread = {
9015 + ...thread,
9016 + limitRecovery: {
9017 + ...recovery,
9018 + autoResume: false,
9019 + requestId: input.command.commandId,
9020 + },
9021 + updatedAt: input.now,
9022 + };
9023 + yield* emitEvent({
9024 + type: "thread.metadata-updated",
9025 + threadId: thread.id,
9026 + occurredAt: input.now,
9027 + payload: thread,
9028 + });
9029 + }
9030 const queuedRuns = runs.filter((run) => run.status === "queued");
9031 const queuedMessages =
9032 queuedRuns.length === 0
@@ -11760,6 +11808,26 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
11808 awaitRestartContinuation: (run) =>
11809 effectOutbox.awaitSettled(`effect:restart-continuation:${run.id}`).pipe(Effect.ignore),
11810 dispatch: dispatchWithReceipt,
11811 + autoArmLimitRecovery: (input) =>
11812 + threadDispatch.withLock(
11813 + input.threadId,
11814 + Effect.gen(function* () {
11815 + const thread = yield* projectionStore
11816 + .getThread(input.threadId)
11817 + .pipe(
11818 + Effect.mapError(
11819 + (cause) => new OrchestratorProjectionError({ threadId: input.threadId, cause }),
11820 + ),
11821 + );
11822 + // Stop or an explicit recovery choice can commit after the sweep reads its candidate.
11823 + if (
11824 + thread.limitRecovery?.runId === input.limitRecovery.runId &&
11825 + thread.limitRecovery.resetAt === input.limitRecovery.resetAt
11826 + )
11827 + return;
11828 + yield* dispatchWithReceiptEffect({ type: "thread.metadata.update", ...input });
11829 + }),
11830 + ),
11831 searchThreadStream: (input) =>
11832 projectionStore
11833 .searchThreadStream(input)
@@ -11909,6 +11977,8 @@ const layerUnavailable: Layer.Layer<OrchestratorV2> = Layer.succeed(
11977 delegatedTaskResultPending: () => Effect.succeed(false),
11978 restartContinuationPending: () => Effect.succeed(false),
11979 awaitRestartContinuation: () => Effect.void,
11980 + autoArmLimitRecovery: (input) =>
11981 + Effect.fail(new OrchestratorProjectionError({ threadId: input.threadId })),
11982 dispatch: (command) =>
11983 Effect.fail(
11984 new OrchestratorDispatchError({
apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts
+13 -1
@@ -5,6 +5,7 @@ import * as Layer from "effect/Layer";
5 import * as Scheduler from "../scheduling/Scheduler.ts";
6 import * as ServerSettings from "../serverSettings.ts";
7 import * as ProjectionStore from "./ProjectionStore.ts";
8 +import * as Orchestrator from "./Orchestrator.ts";
9 import * as ThreadManagement from "./ThreadManagementService.ts";
10
11 export function limitRecoveryCommand(
@@ -73,6 +74,7 @@ export function limitRecoveryCommand(
74 const makeSweep = Effect.gen(function* () {
75 const projections = yield* ProjectionStore.ProjectionStoreV2;
76 const threads = yield* ThreadManagement.ThreadManagementService;
77 + const orchestrator = yield* Orchestrator.OrchestratorV2;
78 const settings = yield* ServerSettings.ServerSettingsService;
79 return Effect.fn("UsageLimitRecoveryWorker.sweep")(function* () {
80 const preferences = yield* settings.getSettings;
@@ -91,7 +93,17 @@ const makeSweep = Effect.gen(function* () {
93 preferences.snoozeLimitedThreads,
94 );
95 if (command === null) continue;
94 - yield* threads.dispatch(command).pipe(
96 + yield* Effect.gen(function* () {
97 + if (command.type === "thread.metadata.update" && command.limitRecovery != null) {
98 + yield* orchestrator.autoArmLimitRecovery({
99 + commandId: command.commandId,
100 + threadId: command.threadId,
101 + limitRecovery: command.limitRecovery,
102 + });
103 + } else {
104 + yield* threads.dispatch(command);
105 + }
106 + }).pipe(
107 Effect.catchCause((cause) =>
108 Effect.logWarning("orchestration-v2.limit-recovery.dispatch-failed", {
109 threadId: thread.id,
apps/server/src/orchestration-v2/runtimeLayer.test.ts
+101 -3
@@ -6198,6 +6198,9 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
6198 "independent-patches",
6199 "expired-snooze",
6200 "wake",
6201 + "stop",
6202 + "stop-before-arm",
6203 + "stop-before-arm-mismatched-recovery",
6204 ] as const)("guards a scheduled usage-limit continuation against %s", (scenario) =>
6205 Effect.gen(function* () {
6206 const orchestrator = yield* Orchestrator.OrchestratorV2;
@@ -6354,8 +6357,68 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
6357 ].includes(scenario);
6358 const autoResume = scenario !== "snooze-only" && scenario !== "wake";
6359 const arm = limitRecoveryCommand(shell, autoResume, DateTime.toEpochMillis(now), snooze);
6357 - assert.isNotNull(arm);
6358 - yield* orchestrator.dispatch(arm!);
6360 + assert(arm?.type === "thread.metadata.update" && arm.limitRecovery != null);
6361 + const automaticArm = {
6362 + commandId: arm.commandId,
6363 + threadId,
6364 + limitRecovery: arm.limitRecovery,
6365 + };
6366 + const stopBeforeArm =
6367 + scenario === "stop-before-arm" || scenario === "stop-before-arm-mismatched-recovery";
6368 + if (stopBeforeArm) {
6369 + if (scenario === "stop-before-arm-mismatched-recovery") {
6370 + const current = yield* orchestrator.getThreadProjection(threadId);
6371 + yield* events.write({
6372 + events: [
6373 + {
6374 + id: EventId.make(`recovery:older-window:${scenario}`),
6375 + type: "thread.metadata-updated",
6376 + threadId,
6377 + occurredAt: now,
6378 + payload: {
6379 + ...current.thread,
6380 + limitRecovery: {
6381 + runId: run.id,
6382 + resetAt: DateTime.formatIso(DateTime.add(now, { minutes: 2 })),
6383 + autoResume: true,
6384 + snooze: false,
6385 + },
6386 + },
6387 + },
6388 + ],
6389 + });
6390 + }
6391 + yield* orchestrator.dispatch({
6392 + type: "thread.stop",
6393 + commandId: CommandId.make(`recovery:stop:${scenario}`),
6394 + threadId,
6395 + });
6396 + const projections = yield* ProjectionStore.ProjectionStoreV2;
6397 + assert.isUndefined(
6398 + (yield* projections.getLimitRecoveryCandidates({
6399 + now,
6400 + autoResume: true,
6401 + snooze: false,
6402 + })).find((row) => row.id === threadId),
6403 + );
6404 + const stoppedShell = (yield* orchestrator.getShellSnapshot()).threads.find(
6405 + (thread) => thread.id === threadId,
6406 + )!;
6407 + assert.isNull(limitRecoveryCommand(stoppedShell, true, DateTime.toEpochMillis(now)));
6408 + yield* orchestrator.autoArmLimitRecovery(automaticArm);
6409 + assert.deepEqual((yield* orchestrator.getThreadProjection(threadId)).thread.limitRecovery, {
6410 + runId: run.id,
6411 + resetAt,
6412 + autoResume: false,
6413 + snooze: false,
6414 + requestId: CommandId.make(`recovery:stop:${scenario}`),
6415 + });
6416 + }
6417 + const armCommand = stopBeforeArm
6418 + ? { ...arm, commandId: CommandId.make(`recovery:explicit-rearm:${scenario}`) }
6419 + : arm;
6420 + if (stopBeforeArm) yield* orchestrator.dispatch(armCommand);
6421 + else yield* orchestrator.autoArmLimitRecovery(automaticArm);
6422 let armedShell = (yield* orchestrator.getShellSnapshot()).threads.find(
6423 (thread) => thread.id === threadId,
6424 )!;
@@ -6364,7 +6427,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
6427 resetAt,
6428 autoResume,
6429 snooze,
6367 - requestId: arm!.commandId,
6430 + requestId: armCommand.commandId,
6431 });
6432 if (snooze)
6433 assert.equal(DateTime.toEpochMillis(armedShell.snoozedUntil!), Date.parse(resetAt));
@@ -6568,6 +6631,17 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
6631 commandId: CommandId.make(`recovery:settle:${scenario}`),
6632 threadId,
6633 });
6634 + if (scenario === "stop") {
6635 + yield* orchestrator.dispatch({
6636 + type: "thread.stop",
6637 + commandId: CommandId.make(`recovery:stop:${scenario}`),
6638 + threadId,
6639 + });
6640 + const stoppedShell = (yield* orchestrator.getShellSnapshot()).threads.find(
6641 + (thread) => thread.id === threadId,
6642 + )!;
6643 + assert.deepInclude(stoppedShell.limitRecovery, { runId: run.id, autoResume: false });
6644 + }
6645 if (scenario === "replacement") {
6646 const current = yield* orchestrator.getThreadProjection(threadId);
6647 const error = current.turnItems.find((item) => item.type === "error")!;
@@ -6626,6 +6700,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
6700 after.runs,
6701 before.runs.length +
6702 (scenario === "resume" ||
6703 + stopBeforeArm ||
6704 scenario === "queued-resume" ||
6705 scenario === "snooze-resume" ||
6706 scenario === "wake-preserve-resume" ||
@@ -6638,6 +6713,7 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
6713 after.messages,
6714 before.messages.length +
6715 (scenario === "resume" ||
6716 + stopBeforeArm ||
6717 scenario === "queued-resume" ||
6718 scenario === "snooze-resume" ||
6719 scenario === "wake-preserve-resume" ||
@@ -6646,6 +6722,28 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
6722 ? 1
6723 : 0),
6724 );
6725 + if (scenario === "stop") {
6726 + assert.deepInclude(after.thread.limitRecovery, { runId: run.id, autoResume: false });
6727 + const projections = yield* ProjectionStore.ProjectionStoreV2;
6728 + const sweep = { now: yield* DateTime.now, autoResume: true, snooze: false };
6729 + assert.isUndefined(
6730 + (yield* projections.getLimitRecoveryCandidates(sweep)).find((row) => row.id === threadId),
6731 + );
6732 + yield* orchestrator.dispatch({
6733 + type: "thread.metadata.update",
6734 + commandId: CommandId.make("recovery:rearm-after-stop"),
6735 + threadId,
6736 + limitRecovery: { runId: run.id, resetAt, autoResume: true },
6737 + });
6738 + const rearmed = (yield* projections.getLimitRecoveryCandidates(sweep)).find(
6739 + (row) => row.id === threadId,
6740 + )!;
6741 + assert.isDefined(rearmed);
6742 + const freshResume = limitRecoveryCommand(rearmed, true, DateTime.toEpochMillis(sweep.now));
6743 + assert.isNotNull(freshResume);
6744 + yield* orchestrator.dispatch(freshResume!);
6745 + assert.lengthOf((yield* orchestrator.getThreadProjection(threadId)).runs, 2);
6746 + }
6747 if (scenario === "queued-resume") {
6748 const held = after.runs[1]!;
6749 assert.equal(held.status, "queued");
apps/server/src/orchestration-v2/runtimeLayer.ts
+7 -1
@@ -368,7 +368,13 @@ export const layerProduction = Layer.mergeAll(
368 layerScheduledTaskProvided,
369 layerSecretRequestsProvided,
370 UsageLimitRecoveryWorker.layer.pipe(
371 - Layer.provide(Layer.mergeAll(ProjectionStore.layer, layerThreadManagementProvided)),
371 + Layer.provide(
372 + Layer.mergeAll(
373 + ProjectionStore.layer,
374 + layerThreadManagementProvided,
375 + layerOrchestratorProvided,
376 + ),
377 + ),
378 ),
379 layerProviderContinuationWorkerProvided,
380 layerAgentSessionImporterProvided,
apps/server/src/scheduling/Scheduler.integration.test.ts
+2
@@ -19,6 +19,7 @@ import * as SqlClient from "effect/sql/SqlClient";
19 import * as TestClock from "effect/testing/TestClock";
20
21 import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts";
22 +import * as Orchestrator from "../orchestration-v2/Orchestrator.ts";
23 import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts";
24 import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts";
25 import * as UsageLimitRecoveryWorker from "../orchestration-v2/UsageLimitRecoveryWorker.ts";
@@ -89,6 +90,7 @@ it.effect.each(["on time", "after restart"])(
90 const receipts = yield* Queue.unbounded<"task" | "retry">();
91 const layerDependencies = Layer.mergeAll(
92 NodeCrypto.layer,
93 + Layer.mock(Orchestrator.OrchestratorV2)({}),
94 Layer.mock(ThreadLaunchService.ThreadLaunchService)({
95 launch: () =>
96 Queue.offer(receipts, "task").pipe(
docs/user/thread-sidebar.md
+2 -2
@@ -206,8 +206,8 @@ continuation. You can cancel it from the thread. Enable **Auto-resume limited
206 threads** in **Settings → General** on web and desktop, or **Settings → Thread
207 behavior** on mobile, to schedule limit stops by default.
208 The environment must be running when the reset arrives; it resumes overdue
209 -continuations after a restart. Sending a new message, archiving, or settling the
210 -thread prevents a pending continuation from starting.
209 +continuations after a restart. Stopping the thread, sending a new message,
210 +archiving, or settling it prevents a pending continuation from starting.
211
212 Choose **Snooze until reset** to hide the thread until its allowance returns.
213 Snooze and auto-resume are independent: snooze alone wakes the thread without