The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 08:15 UTC
fix(server): interrupted scheduled task runs no longer stay busy (#28474)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Theo Browne committed
Nov 1, 2026 at 00:03 UTC
ce9e01640636c0884c4a7574a722bf1d773cfa07
3 files changed
+220
-92
apps/server/src/scheduledTasks/ScheduledTaskService.test.ts
+69
-1
@@ -2,7 +2,7 @@ import * as Scheduler from "../scheduling/Scheduler.ts";
2
import * as NodeUtil from "node:util";
3
4
import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
5
-import { assert, it } from "@effect/vitest";
5
+import { assert, it, vi } from "@effect/vitest";
6
import {
7
ProjectId,
8
ProviderInstanceId,
@@ -12,6 +12,7 @@ import {
12
} from "@t3tools/contracts";
13
import * as DateTime from "effect/DateTime";
14
import * as Effect from "effect/Effect";
15
+import * as Exit from "effect/Exit";
16
import * as Layer from "effect/Layer";
17
import * as Logger from "effect/Logger";
18
import * as SqlClient from "effect/sql/SqlClient";
@@ -476,3 +477,70 @@ it.effect("a bound run follows its thread when promotion moves it mid-send", ()
477
}).pipe(Effect.provide(layer));
478
}).pipe(Effect.provide(SqlitePersistence.layerMemory)),
479
);
480
+
481
+it.effect("a run interrupted right after it reserves the task does not block the next run", () =>
482
+ Effect.gen(function* () {
483
+ const launches = yield* Ref.make(0);
484
+ const layer = ScheduledTaskService.layer.pipe(
485
+ Layer.provide(
486
+ Layer.mergeAll(
487
+ Layer.mock(ThreadLaunchService.ThreadLaunchService)({
488
+ launch: () =>
489
+ Ref.update(launches, (n) => n + 1).pipe(
490
+ Effect.as({ threadId: ThreadId.make("thread:launched"), resumed: false } as never),
491
+ ),
492
+ }),
493
+ Layer.mock(ThreadManagementService.ThreadManagementService)({}),
494
+ Layer.mock(SecretRequests.SecretRequests)({}),
495
+ Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({
496
+ namedProjectsRoot: "/projects",
497
+ isScratchProject: () => Effect.succeed(false),
498
+ }),
499
+ NodeCrypto.layer,
500
+ Layer.succeed(Scheduler.Scheduler, { register: () => Effect.void }),
501
+ ),
502
+ ),
503
+ );
504
+ yield* Effect.gen(function* () {
505
+ const service = yield* ScheduledTaskService.ScheduledTaskService;
506
+ const { task } = yield* service.upsert({
507
+ id: ScheduledTaskId.make("task:interrupted-reservation"),
508
+ title: "Check in",
509
+ prompt: "Check progress",
510
+ enabled: true,
511
+ schedule: { type: "interval", everyMs: 60_000 },
512
+ projectId: ProjectId.make("project:test"),
513
+ workspaceStrategy: { type: "root" },
514
+ modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" },
515
+ runtimeMode: "full-access",
516
+ interactionMode: "default",
517
+ });
518
+
519
+ // The reservation is the first Set.add of the task id; interrupt the run right there.
520
+ const add = Set.prototype.add;
521
+ let interrupted = false;
522
+ const first = yield* Effect.gen(function* () {
523
+ using _reserve = vi.spyOn(Set.prototype, "add").mockImplementation(function (
524
+ this: Set<unknown>,
525
+ value: unknown,
526
+ ) {
527
+ const result = add.call(this, value);
528
+ if (value === task.id && !interrupted) {
529
+ interrupted = true;
530
+ Fiber.getCurrent()?.interruptUnsafe();
531
+ }
532
+ return result;
533
+ });
534
+ return yield* service
535
+ .runNow({ id: task.id })
536
+ .pipe(Effect.forkChild, Effect.flatMap(Fiber.await));
537
+ });
538
+ assert.isTrue(interrupted);
539
+ assert.isTrue(Exit.hasInterrupts(first));
540
+
541
+ const second = yield* service.runNow({ id: task.id });
542
+ assert.equal(second.threadId, ThreadId.make("thread:launched"));
543
+ assert.equal(yield* Ref.get(launches), 1);
544
+ }).pipe(Effect.provide(layer));
545
+ }).pipe(Effect.provide(SqlitePersistence.layerMemory)),
546
+);
apps/server/src/scheduledTasks/ScheduledTaskService.ts
+99
-91
@@ -786,20 +786,7 @@ export const layer = Layer.effect(
786
// Read before reserving: nothing between the reservation and its release may fail.
787
const startedAt = yield* localNow;
788
const startedAtIso = iso(startedAt);
789
- const reserved = yield* Ref.modify(activeRuns, (active) => {
790
- if (active.has(task.id)) return [false, active] as const;
791
- const next = new Set(active);
792
- next.add(task.id);
793
- return [true, next] as const;
794
- });
795
- if (!reserved) {
796
- if (trigger !== "scheduled") {
797
- return yield* taskError("Schedule task is already running.", { taskId: task.id });
798
- }
799
- return { task, threadId: null, dispatchFailed: false };
800
- }
801
-
802
- return yield* Effect.gen(function* () {
789
+ const run = Effect.gen(function* () {
790
// Re-read stale poll snapshots so deleted, paused or postponed tasks cannot fire.
791
const active = yield* findTask(task.id);
792
if (active === null) {
@@ -1018,13 +1005,30 @@ export const layer = Layer.effect(
1005
Effect.onError((cause) =>
1006
releaseStuckRun(task, startedAtIso, errorMessage(cause), webhook?.deliveryId),
1007
),
1021
- Effect.ensuring(
1022
- Ref.update(activeRuns, (active) => {
1023
- const next = new Set(active);
1024
- next.delete(task.id);
1025
- return next;
1026
- }),
1027
- ),
1008
+ );
1009
+
1010
+ // Reserve and release as one bracket so an interrupt cannot leave the task marked busy.
1011
+ return yield* Effect.acquireUseRelease(
1012
+ Ref.modify(activeRuns, (active) => {
1013
+ if (active.has(task.id)) return [false, active] as const;
1014
+ const next = new Set(active);
1015
+ next.add(task.id);
1016
+ return [true, next] as const;
1017
+ }),
1018
+ (reserved) =>
1019
+ reserved
1020
+ ? run
1021
+ : trigger !== "scheduled"
1022
+ ? Effect.fail(taskError("Schedule task is already running.", { taskId: task.id }))
1023
+ : Effect.succeed({ task, threadId: null, dispatchFailed: false }),
1024
+ (reserved) =>
1025
+ reserved
1026
+ ? Ref.update(activeRuns, (active) => {
1027
+ const next = new Set(active);
1028
+ next.delete(task.id);
1029
+ return next;
1030
+ })
1031
+ : Effect.void,
1032
);
1033
});
1034
@@ -2243,83 +2247,87 @@ export const layer = Layer.effect(
2247
}
2248
yield* Effect.forEach(
2249
tasks,
2246
- ({ task_id: taskId }) =>
2247
- Effect.gen(function* () {
2248
- const reserved = yield* Ref.modify(pendingInFlight, (active) =>
2249
- active.has(taskId)
2250
- ? ([false, active] as const)
2251
- : ([true, new Set(active).add(taskId)] as const),
2252
- );
2253
- if (!reserved) return;
2254
- const permit = yield* webhookPermit(taskId);
2255
- // One worker per task waits on its permit, then drains persisted rows in order.
2256
- // Other tasks remain independent and ticks never reload an active task's payloads.
2257
- const dispatch = Effect.gen(function* () {
2258
- while (true) {
2259
- const [entry] = yield* sql<{
2260
- delivery_id: string;
2261
- task_json: string;
2262
- request_json: string;
2263
- state: string;
2264
- }>`
2250
+ ({ task_id: taskId }) => {
2251
+ // One worker per task waits on its permit, then drains persisted rows in order.
2252
+ // Other tasks remain independent and ticks never reload an active task's payloads.
2253
+ const dispatch = Effect.gen(function* () {
2254
+ while (true) {
2255
+ const [entry] = yield* sql<{
2256
+ delivery_id: string;
2257
+ task_json: string;
2258
+ request_json: string;
2259
+ state: string;
2260
+ }>`
2261
SELECT delivery_id, task_json, request_json, state FROM scheduled_task_webhook_pending
2262
WHERE task_id = ${taskId} AND state IN ('dispatching', 'running', 'completed', 'failed') ORDER BY due_at, rowid LIMIT 1
2263
`.pipe(NodeSqliteClient.readOnly);
2268
- if (entry === undefined) return;
2269
- const remove = sql`DELETE FROM scheduled_task_webhook_pending WHERE delivery_id = ${entry.delivery_id}`;
2270
- if (entry.state === "completed" || entry.state === "failed") {
2271
- if (entry.state === "failed") {
2272
- yield* markDeliveryFailed(entry.delivery_id, "The run failed to start.");
2273
- yield* releaseFailedDelivery(entry.delivery_id);
2274
- }
2275
- yield* remove;
2276
- continue;
2277
- }
2278
- const prepared = yield* Effect.gen(function* () {
2279
- const task = yield* decodePendingTask(entry.task_json);
2280
- const request = yield* decodePendingRequest(entry.request_json);
2281
- const frameId = Buffer.from(yield* crypto.randomBytes(6)).toString("hex");
2282
- return { task, rendered: renderWebhookPrompt(task.prompt, request, frameId) };
2283
- }).pipe(Effect.exit);
2284
- if (Exit.isFailure(prepared)) {
2285
- if (Cause.hasInterruptsOnly(prepared.cause))
2286
- return yield* Effect.failCause(prepared.cause);
2287
- yield* markDeliveryFailed(
2288
- entry.delivery_id,
2289
- "The pending webhook payload could not be prepared.",
2290
- );
2291
- yield* releaseFailedDelivery(entry.delivery_id);
2292
- yield* remove;
2293
- continue;
2294
- }
2295
- const { task, rendered } = prepared.value;
2296
- if (rendered.prompt.trim().length > PROVIDER_SEND_TURN_MAX_INPUT_CHARS) {
2297
- yield* markDeliveryFailed(entry.delivery_id, PROMPT_TOO_LONG);
2264
+ if (entry === undefined) return;
2265
+ const remove = sql`DELETE FROM scheduled_task_webhook_pending WHERE delivery_id = ${entry.delivery_id}`;
2266
+ if (entry.state === "completed" || entry.state === "failed") {
2267
+ if (entry.state === "failed") {
2268
+ yield* markDeliveryFailed(entry.delivery_id, "The run failed to start.");
2269
yield* releaseFailedDelivery(entry.delivery_id);
2299
- } else {
2300
- yield* sql`UPDATE scheduled_task_webhook_deliveries SET outcome = 'accepted', error = NULL, rendered_prompt = ${rendered.prompt.slice(0, WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT)}, missing_fields_json = ${encodeMissingFieldsJson(rendered.missing)} WHERE delivery_id = ${entry.delivery_id}`;
2301
- if (!(yield* runWebhookDelivery(task, entry.delivery_id, rendered.prompt)))
2302
- return;
2270
}
2271
yield* remove;
2272
+ continue;
2273
}
2306
- });
2307
- yield* dispatch.pipe(
2308
- permit.withPermits(1),
2309
- Effect.catch((cause) =>
2310
- Effect.logWarning("Pending webhook dispatch failed", { cause }),
2311
- ),
2312
- Effect.ensuring(
2313
- Ref.update(pendingInFlight, (active) => {
2314
- const next = new Set(active);
2315
- next.delete(taskId);
2316
- return next;
2317
- }),
2318
- ),
2319
- Effect.withSpan("ScheduledTaskService.dispatchPendingWebhook", { root: true }),
2320
- Effect.forkIn(serviceScope),
2321
- );
2322
- }),
2274
+ const prepared = yield* Effect.gen(function* () {
2275
+ const task = yield* decodePendingTask(entry.task_json);
2276
+ const request = yield* decodePendingRequest(entry.request_json);
2277
+ const frameId = Buffer.from(yield* crypto.randomBytes(6)).toString("hex");
2278
+ return { task, rendered: renderWebhookPrompt(task.prompt, request, frameId) };
2279
+ }).pipe(Effect.exit);
2280
+ if (Exit.isFailure(prepared)) {
2281
+ if (Cause.hasInterruptsOnly(prepared.cause))
2282
+ return yield* Effect.failCause(prepared.cause);
2283
+ yield* markDeliveryFailed(
2284
+ entry.delivery_id,
2285
+ "The pending webhook payload could not be prepared.",
2286
+ );
2287
+ yield* releaseFailedDelivery(entry.delivery_id);
2288
+ yield* remove;
2289
+ continue;
2290
+ }
2291
+ const { task, rendered } = prepared.value;
2292
+ if (rendered.prompt.trim().length > PROVIDER_SEND_TURN_MAX_INPUT_CHARS) {
2293
+ yield* markDeliveryFailed(entry.delivery_id, PROMPT_TOO_LONG);
2294
+ yield* releaseFailedDelivery(entry.delivery_id);
2295
+ } else {
2296
+ yield* sql`UPDATE scheduled_task_webhook_deliveries SET outcome = 'accepted', error = NULL, rendered_prompt = ${rendered.prompt.slice(0, WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT)}, missing_fields_json = ${encodeMissingFieldsJson(rendered.missing)} WHERE delivery_id = ${entry.delivery_id}`;
2297
+ if (!(yield* runWebhookDelivery(task, entry.delivery_id, rendered.prompt))) return;
2298
+ }
2299
+ yield* remove;
2300
+ }
2301
+ });
2302
+ // The worker owns its reservation, so an interrupt anywhere still releases it.
2303
+ return Effect.acquireUseRelease(
2304
+ Ref.modify(pendingInFlight, (active) =>
2305
+ active.has(taskId)
2306
+ ? ([false, active] as const)
2307
+ : ([true, new Set(active).add(taskId)] as const),
2308
+ ),
2309
+ (reserved) =>
2310
+ reserved
2311
+ ? webhookPermit(taskId).pipe(
2312
+ Effect.flatMap((permit) => dispatch.pipe(permit.withPermits(1))),
2313
+ Effect.catch((cause) =>
2314
+ Effect.logWarning("Pending webhook dispatch failed", { cause }),
2315
+ ),
2316
+ Effect.withSpan("ScheduledTaskService.dispatchPendingWebhook", {
2317
+ root: true,
2318
+ }),
2319
+ )
2320
+ : Effect.void,
2321
+ (reserved) =>
2322
+ reserved
2323
+ ? Ref.update(pendingInFlight, (active) => {
2324
+ const next = new Set(active);
2325
+ next.delete(taskId);
2326
+ return next;
2327
+ })
2328
+ : Effect.void,
2329
+ ).pipe(Effect.forkIn(serviceScope));
2330
+ },
2331
{ discard: true },
2332
);
2333
});
apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts
+52
@@ -12,6 +12,7 @@ import * as DateTime from "effect/DateTime";
12
import * as Crypto from "effect/Crypto";
13
import * as Deferred from "effect/Deferred";
14
import * as Effect from "effect/Effect";
15
+import * as Exit from "effect/Exit";
16
import * as Fiber from "effect/Fiber";
17
import * as Layer from "effect/Layer";
18
import * as Queue from "effect/Queue";
@@ -1005,6 +1006,57 @@ it.effect("recovers a persisted coalescing window after restarting the service",
1006
);
1007
});
1008
1009
+it.effect("a pending dispatch interrupted at its permit does not block later polls", () => {
1010
+ const workers = new Map<string, Effect.Effect<void>>();
1011
+ // One shared dispatcher lets the test run the retry's forked worker to completion.
1012
+ const mixed = new EffectScheduler.MixedScheduler("sync");
1013
+ const dispatcher = mixed.makeDispatcher();
1014
+ return withService(
1015
+ ({ service, sql, launches }) =>
1016
+ Effect.gen(function* () {
1017
+ const { task } = yield* service.upsert(
1018
+ yield* webhookTaskInput({
1019
+ schedule: {
1020
+ type: "webhook",
1021
+ dedupe: { coalesceKey: "{{headers.content-type}}", coalesceWindowSeconds: 10 },
1022
+ },
1023
+ }),
1024
+ );
1025
+ yield* service.triggerWebhook(requestFor(task));
1026
+ yield* TestClock.adjust("10 seconds");
1027
+ const poll = workers.get("webhook-coalescing")!;
1028
+
1029
+ const entered = yield* Deferred.make<Fiber.Fiber<unknown, unknown>>();
1030
+ const interruptAtPermit = Tracer.make({
1031
+ span: (options) => {
1032
+ if (options.name === "ScheduledTaskService.webhookPermit") {
1033
+ const fiber = Fiber.getCurrent()!;
1034
+ Deferred.doneUnsafe(entered, Effect.succeed(fiber));
1035
+ fiber.interruptUnsafe();
1036
+ }
1037
+ return new Tracer.NativeSpan(options);
1038
+ },
1039
+ });
1040
+ const firstPoll = yield* poll.pipe(Effect.withTracer(interruptAtPermit), Effect.forkChild);
1041
+ assert.isTrue(Exit.hasInterrupts(yield* Fiber.await(yield* Deferred.await(entered))));
1042
+ yield* Fiber.await(firstPoll);
1043
+
1044
+ yield* poll;
1045
+ dispatcher.flush();
1046
+
1047
+ assert.equal(yield* Queue.size(launches), 1);
1048
+ assert.lengthOf(yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`, 0);
1049
+ }),
1050
+ { workers },
1051
+ ).pipe(
1052
+ Effect.provideService(EffectScheduler.Scheduler, {
1053
+ executionMode: mixed.executionMode,
1054
+ makeDispatcher: () => dispatcher,
1055
+ shouldYield: (fiber) => mixed.shouldYield(fiber),
1056
+ }),
1057
+ );
1058
+});
1059
+
1060
it.effect("rolls back replay and pending state when recording a delivery fails", () => {
1061
const workers = new Map<string, Effect.Effect<void>>();
1062
return withService(