@devries-cameron20 / t3code Mirror · read-only / commits / e496db3a5e
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 10:45 UTC

fix(server): a task cancelled as its usage limit lands no longer resumes at the reset (#24634)

Co-authored-by: maria-rcks <254055478+maria-rcks@users.noreply.github.com>

spoukyii committed Oct 30, 2026 at 14:27 UTC e496db3a5e1484431112866c977c8649aa634e2a
2 files changed +364 -1
apps/server/src/mcp/TaskCancelUsageLimit.integration.test.ts new
+353
@@ -0,0 +1,353 @@
1 +import * as NodeServices from "@effect/platform-node/NodeServices";
2 +import { assert, it } from "@effect/vitest";
3 +import {
4 + CommandId,
5 + EnvironmentId,
6 + MessageId,
7 + ProjectId,
8 + ProviderDriverKind,
9 + ProviderInstanceId,
10 + ProviderThreadId,
11 + ProviderTurnId,
12 + ThreadId,
13 + type OrchestrationV2DomainEvent,
14 +} from "@t3tools/contracts";
15 +import * as DateTime from "effect/DateTime";
16 +import * as Effect from "effect/Effect";
17 +import * as Fiber from "effect/Fiber";
18 +import * as Layer from "effect/Layer";
19 +import * as Queue from "effect/Queue";
20 +import * as Stream from "effect/Stream";
21 +import * as TestClock from "effect/testing/TestClock";
22 +
23 +import { ClaudeProviderCapabilitiesV2 } from "../orchestration-v2/Adapters/ClaudeAdapterV2.ts";
24 +import * as CommandReceiptStore from "../orchestration-v2/CommandReceiptStore.ts";
25 +import * as EffectWorker from "../orchestration-v2/EffectWorker.ts";
26 +import * as Orchestrator from "../orchestration-v2/Orchestrator.ts";
27 +import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts";
28 +import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
29 +import * as ProviderAdapterRegistry from "../orchestration-v2/ProviderAdapterRegistry.ts";
30 +import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts";
31 +import * as UsageLimitRecoveryWorker from "../orchestration-v2/UsageLimitRecoveryWorker.ts";
32 +import * as GitVcsDriver from "../vcs/GitVcsDriver.ts";
33 +import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts";
34 +import * as ServerSettings from "../serverSettings.ts";
35 +import * as ProviderReplayHarness from "../orchestration-v2/testkit/ProviderReplayHarness.ts";
36 +import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace";
37 +import * as SqlitePersistence from "../persistence/Sqlite.ts";
38 +import * as ProjectService from "../project/ProjectService.ts";
39 +import * as ProviderRegistry from "../provider/ProviderRegistry.ts";
40 +import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts";
41 +import * as Scheduler from "../scheduling/Scheduler.ts";
42 +import * as SecretRequests from "../secrets/SecretRequests.ts";
43 +import * as RemoteDelegation from "../peer/RemoteDelegation.ts";
44 +import type { McpInvocationScope } from "./McpInvocationContext.ts";
45 +import * as OrchestratorMcpService from "./OrchestratorMcpService.ts";
46 +
47 +const driver = ProviderDriverKind.make("claudeAgent");
48 +const instanceId = ProviderInstanceId.make("claudeAgent");
49 +const modelSelection = { instanceId, model: "claude-test" };
50 +
51 +// A delegated child that fails on a usage limit is armed to continue at the reset.
52 +// Once its parent cancels the task, nothing is waiting for that continuation.
53 +it.effect.each([
54 + { cancel: "after its limit", cancelStatus: "failed", resumed: false },
55 + { cancel: "as its limit lands", cancelStatus: "cancel_requested", resumed: false },
56 + { cancel: "never", cancelStatus: null, resumed: true },
57 +] as const)(
58 + "a delegated child whose task is cancelled $cancel resumes at the reset: $resumed",
59 + ({ cancel, cancelStatus, resumed }) =>
60 + Effect.scoped(
61 + Effect.gen(function* () {
62 + const cwd = yield* checkpointWorkspace("task-cancel-usage-limit");
63 + // Set while the provider's usage-limit result is already on its way.
64 + let limitInFlight: Effect.Effect<void> = Effect.void;
65 + const turns: Array<{
66 + readonly turn: ProviderAdapter.ProviderAdapterV2TurnInput;
67 + readonly events: Queue.Queue<ProviderAdapter.ProviderAdapterV2Event>;
68 + }> = [];
69 + const adapter: ProviderAdapter.ProviderAdapterV2["Service"] = {
70 + instanceId,
71 + driver,
72 + getCapabilities: () => Effect.succeed(ClaudeProviderCapabilitiesV2),
73 + planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" }),
74 + openSession: (input) =>
75 + Effect.gen(function* () {
76 + const now = yield* DateTime.now;
77 + const events = yield* Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event>();
78 + return {
79 + instanceId,
80 + driver,
81 + providerSessionId: input.providerSessionId,
82 + providerSession: {
83 + id: input.providerSessionId,
84 + driver,
85 + providerInstanceId: instanceId,
86 + status: "ready",
87 + cwd,
88 + model: modelSelection.model,
89 + capabilities: ClaudeProviderCapabilitiesV2,
90 + createdAt: now,
91 + updatedAt: now,
92 + lastError: null,
93 + },
94 + events: Stream.fromQueue(events),
95 + ensureThread: ({ threadId }) =>
96 + Effect.succeed({
97 + id: ProviderThreadId.make(`provider-thread:${threadId}`),
98 + driver,
99 + providerInstanceId: instanceId,
100 + providerSessionId: input.providerSessionId,
101 + appThreadId: threadId,
102 + ownerNodeId: null,
103 + nativeThreadRef: { driver, nativeId: `native:${threadId}`, strength: "strong" },
104 + nativeConversationHeadRef: null,
105 + status: "idle",
106 + firstRunOrdinal: null,
107 + lastRunOrdinal: null,
108 + handoffIds: [],
109 + forkedFrom: null,
110 + createdAt: now,
111 + updatedAt: now,
112 + }),
113 + resumeThread: ({ providerThread }) => Effect.succeed(providerThread),
114 + startTurn: (turn) =>
115 + Effect.gen(function* () {
116 + turns.push({ turn, events });
117 + yield* Queue.offer(events, {
118 + type: "provider_turn.updated",
119 + driver,
120 + providerTurn: {
121 + id: ProviderTurnId.make(`provider-turn:${turn.attemptId}`),
122 + providerThreadId: turn.providerThread.id,
123 + nodeId: turn.rootNodeId,
124 + runAttemptId: turn.attemptId,
125 + nativeTurnRef: null,
126 + ordinal: turn.providerTurnOrdinal,
127 + status: "running",
128 + startedAt: now,
129 + completedAt: null,
130 + },
131 + });
132 + }),
133 + steerTurn: () => Effect.die("unused"),
134 + interruptTurn: () => Effect.suspend(() => limitInFlight),
135 + respondToRuntimeRequest: () => Effect.die("unused"),
136 + readThreadSnapshot: () => Effect.die("unused"),
137 + rollbackThread: () => Effect.die("unused"),
138 + forkThread: () => Effect.die("unused"),
139 + };
140 + }),
141 + };
142 + const layerAdapters = ProviderAdapterRegistry.layerSingle(adapter);
143 + const layerOrchestrator = ProviderReplayHarness.layerWithRegistry(
144 + { name: "task-cancel-usage-limit" },
145 + layerAdapters,
146 + { runEffectWorker: false },
147 + );
148 + const layerOrchestration = Layer.merge(
149 + layerOrchestrator,
150 + ThreadManagementService.layer.pipe(
151 + Layer.provide(layerOrchestrator),
152 + Layer.provide(ServerSettings.layerTest()),
153 + Layer.provide(
154 + CommandReceiptStore.layer.pipe(Layer.provide(SqlitePersistence.layerMemory)),
155 + ),
156 + ),
157 + );
158 + const sweeps = new Map<string, Effect.Effect<void>>();
159 + const layerRecovery = UsageLimitRecoveryWorker.layer.pipe(
160 + Layer.provide(
161 + Layer.mergeAll(
162 + Layer.succeed(Scheduler.Scheduler, {
163 + register: <E, R>(name: string, work: Effect.Effect<void, E, R>) =>
164 + Effect.gen(function* () {
165 + const context = yield* Effect.context<R>();
166 + sweeps.set(name, work.pipe(Effect.provideContext(context), Effect.orDie));
167 + }),
168 + }),
169 + ProjectionStore.layer.pipe(Layer.provide(SqlitePersistence.layerMemory)),
170 + ServerSettings.layerTest({ autoResumeLimitedThreads: true }),
171 + ),
172 + ),
173 + );
174 + const layerTest = Layer.merge(
175 + OrchestratorMcpService.layer.pipe(
176 + Layer.provide(Layer.mock(ThreadLaunchService.ThreadLaunchService)({})),
177 + Layer.provide(Layer.mock(GitVcsDriver.GitVcsDriver)({})),
178 + ),
179 + layerRecovery,
180 + ).pipe(
181 + Layer.provideMerge(layerOrchestration),
182 + Layer.provide(
183 + Layer.mergeAll(
184 + layerAdapters,
185 + Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }),
186 + Layer.mock(ProjectService.ProjectService)({}),
187 + Layer.mock(SecretRequests.SecretRequests)({}),
188 + Layer.mock(RemoteDelegation.RemoteDelegation)({}),
189 + ServerSettings.layerTest(),
190 + Layer.mock(ScheduledTaskService.ScheduledTaskService)({}),
191 + NodeServices.layer,
192 + ),
193 + ),
194 + );
195 +
196 + yield* Effect.gen(function* () {
197 + const orchestrator = yield* Orchestrator.OrchestratorV2;
198 + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
199 + const service = yield* OrchestratorMcpService.OrchestratorMcpService;
200 + const sweep = Effect.andThen(
201 + Effect.suspend(() => sweeps.get("usage-limit-recovery")!),
202 + worker.drain(),
203 + );
204 + const parentThreadId = ThreadId.make("thread:task-cancel-limit-parent");
205 + const watch = (predicate: (event: OrchestrationV2DomainEvent) => boolean) =>
206 + orchestrator.streamDomainEvents.pipe(
207 + Stream.filter(predicate),
208 + Stream.take(1),
209 + Stream.runDrain,
210 + Effect.forkScoped,
211 + );
212 + const turnRunning = () =>
213 + watch(
214 + (event) =>
215 + event.type === "provider-turn.updated" && event.payload.status === "running",
216 + );
217 + yield* orchestrator.dispatch({
218 + type: "thread.create",
219 + commandId: CommandId.make("create-parent"),
220 + threadId: parentThreadId,
221 + projectId: ProjectId.make("project:task-cancel-limit"),
222 + title: "Parent",
223 + modelSelection,
224 + runtimeMode: "full-access",
225 + interactionMode: "default",
226 + branch: null,
227 + worktreePath: cwd,
228 + createdBy: "user",
229 + creationSource: "web",
230 + });
231 + const parentRunning = yield* turnRunning();
232 + yield* orchestrator.dispatch({
233 + type: "message.dispatch",
234 + commandId: CommandId.make("start-parent"),
235 + threadId: parentThreadId,
236 + messageId: MessageId.make("message:start-parent"),
237 + text: "Delegate a review",
238 + attachments: [],
239 + dispatchMode: { type: "start_immediately" },
240 + createdBy: "user",
241 + creationSource: "web",
242 + });
243 + yield* worker.drain();
244 + yield* Fiber.join(parentRunning);
245 +
246 + const parentRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs[0]!;
247 + const childRunning = yield* turnRunning();
248 + yield* orchestrator.dispatch({
249 + type: "delegated_task.request",
250 + commandId: CommandId.make("delegate"),
251 + parentThreadId,
252 + parentRunId: parentRun.id,
253 + parentNodeId: parentRun.rootNodeId!,
254 + task: "Review the change",
255 + modelSelection,
256 + runtimeMode: "full-access",
257 + interactionMode: "default",
258 + completionWake: "always",
259 + createdBy: "agent",
260 + creationSource: "mcp",
261 + });
262 + yield* worker.drain();
263 + yield* Fiber.join(childRunning);
264 + const task = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents[0]!;
265 + const childThreadId = task.childThreadId!;
266 + const child = turns.find((entry) => entry.turn.threadId === childThreadId)!;
267 + const childTurn = (yield* orchestrator.getThreadProjection(childThreadId))
268 + .providerTurns[0]!;
269 +
270 + const scope: McpInvocationScope = {
271 + environmentId: EnvironmentId.make("environment:task-cancel-limit"),
272 + requestNamespace: "provider-session:task-cancel-limit",
273 + thread: {
274 + threadId: parentThreadId,
275 + providerSessionId: "provider-session:task-cancel-limit",
276 + providerInstanceId: instanceId,
277 + },
278 + client: undefined,
279 + capabilities: new Set(["orchestration"]),
280 + issuedAt: 1,
281 + };
282 + const cancelTask = Effect.gen(function* () {
283 + const result = yield* service.cancelTask(scope, {
284 + taskId: task.id,
285 + clientRequestId: "cancel-task",
286 + });
287 + yield* worker.drain();
288 + return result.status;
289 + });
290 +
291 + const now = yield* DateTime.now;
292 + const childFailed = yield* watch(
293 + (event) =>
294 + event.type === "run.updated" &&
295 + event.payload.id === child.turn.runId &&
296 + event.payload.status === "failed",
297 + );
298 + const hitLimit = Effect.forEach(
299 + [
300 + {
301 + type: "provider_turn.updated",
302 + driver,
303 + providerTurn: { ...childTurn, status: "failed", completedAt: now },
304 + },
305 + {
306 + type: "turn.terminal",
307 + driver,
308 + providerThreadId: childTurn.providerThreadId,
309 + providerTurnId: childTurn.id,
310 + runOrdinal: child.turn.runOrdinal,
311 + failureItemOrdinal: 1,
312 + status: "failed",
313 + failure: {
314 + class: "usage_limit",
315 + message: "Claude usage limit reached.",
316 + code: null,
317 + retryable: null,
318 + resetAt: DateTime.formatIso(DateTime.add(now, { minutes: 1 })),
319 + },
320 + threadDisposition: "reusable",
321 + },
322 + ] satisfies ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event>,
323 + (event) => Queue.offer(child.events, event),
324 + { discard: true },
325 + );
326 + let cancelled: string | null = null;
327 + if (cancel === "as its limit lands") {
328 + limitInFlight = hitLimit;
329 + cancelled = yield* cancelTask;
330 + } else {
331 + yield* hitLimit;
332 + }
333 + yield* Fiber.join(childFailed);
334 + yield* worker.drain();
335 +
336 + // The sweep runs every few seconds, so it arms the child before a parent reacts.
337 + yield* sweep;
338 + if (cancel === "after its limit") cancelled = yield* cancelTask;
339 + assert.equal(cancelled, cancelStatus);
340 +
341 + yield* TestClock.adjust("1 minute");
342 + yield* sweep;
343 + const childAfter = yield* orchestrator.getThreadProjection(childThreadId);
344 + assert.deepEqual(
345 + childAfter.messages.flatMap((message) =>
346 + message.role === "user" ? [message.text] : [],
347 + ),
348 + resumed ? ["Review the change", "Continue where you left off."] : ["Review the change"],
349 + );
350 + }).pipe(Effect.provide(layerTest));
351 + }),
352 + ),
353 +);
apps/server/src/orchestration-v2/Orchestrator.ts
+11 -1
@@ -13618,7 +13618,17 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
13618 thread.limitRecovery.resetAt === input.limitRecovery.resetAt
13619 )
13620 return;
13621 - yield* dispatchWithReceiptEffect({ type: "thread.metadata.update", ...input });
13621 + const command = { type: "thread.metadata.update", ...input } as const;
13622 + // A Stop that reached the run before its limit failure landed had nothing to disarm.
13623 + const stopped = yield* stopReachedRun(command, input.threadId, input.limitRecovery.runId);
13624 + yield* dispatchWithReceiptEffect(
13625 + stopped
13626 + ? {
13627 + ...command,
13628 + limitRecovery: { ...input.limitRecovery, autoResume: false, snooze: false },
13629 + }
13630 + : command,
13631 + );
13632 }),
13633 );
13634 }),