@devries-cameron20 / t3code-1 / commits / ce9e016406
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): interrupted scheduled task runs no longer stay busy (#28474)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Theo Browne committed Nov 1, 2026 at 00:03 UTC ce9e01640636c0884c4a7574a722bf1d773cfa07
3 files changed +220 -92
apps/server/src/scheduledTasks/ScheduledTaskService.test.ts
+69 -1
@@ -2,7 +2,7 @@ import * as Scheduler from "../scheduling/Scheduler.ts";
2 import * as NodeUtil from "node:util";
3
4 import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
5 -import { assert, it } from "@effect/vitest";
5 +import { assert, it, vi } from "@effect/vitest";
6 import {
7 ProjectId,
8 ProviderInstanceId,
@@ -12,6 +12,7 @@ import {
12 } from "@t3tools/contracts";
13 import * as DateTime from "effect/DateTime";
14 import * as Effect from "effect/Effect";
15 +import * as Exit from "effect/Exit";
16 import * as Layer from "effect/Layer";
17 import * as Logger from "effect/Logger";
18 import * as SqlClient from "effect/sql/SqlClient";
@@ -476,3 +477,70 @@ it.effect("a bound run follows its thread when promotion moves it mid-send", ()
477 }).pipe(Effect.provide(layer));
478 }).pipe(Effect.provide(SqlitePersistence.layerMemory)),
479 );
480 +
481 +it.effect("a run interrupted right after it reserves the task does not block the next run", () =>
482 + Effect.gen(function* () {
483 + const launches = yield* Ref.make(0);
484 + const layer = ScheduledTaskService.layer.pipe(
485 + Layer.provide(
486 + Layer.mergeAll(
487 + Layer.mock(ThreadLaunchService.ThreadLaunchService)({
488 + launch: () =>
489 + Ref.update(launches, (n) => n + 1).pipe(
490 + Effect.as({ threadId: ThreadId.make("thread:launched"), resumed: false } as never),
491 + ),
492 + }),
493 + Layer.mock(ThreadManagementService.ThreadManagementService)({}),
494 + Layer.mock(SecretRequests.SecretRequests)({}),
495 + Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({
496 + namedProjectsRoot: "/projects",
497 + isScratchProject: () => Effect.succeed(false),
498 + }),
499 + NodeCrypto.layer,
500 + Layer.succeed(Scheduler.Scheduler, { register: () => Effect.void }),
501 + ),
502 + ),
503 + );
504 + yield* Effect.gen(function* () {
505 + const service = yield* ScheduledTaskService.ScheduledTaskService;
506 + const { task } = yield* service.upsert({
507 + id: ScheduledTaskId.make("task:interrupted-reservation"),
508 + title: "Check in",
509 + prompt: "Check progress",
510 + enabled: true,
511 + schedule: { type: "interval", everyMs: 60_000 },
512 + projectId: ProjectId.make("project:test"),
513 + workspaceStrategy: { type: "root" },
514 + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" },
515 + runtimeMode: "full-access",
516 + interactionMode: "default",
517 + });
518 +
519 + // The reservation is the first Set.add of the task id; interrupt the run right there.
520 + const add = Set.prototype.add;
521 + let interrupted = false;
522 + const first = yield* Effect.gen(function* () {
523 + using _reserve = vi.spyOn(Set.prototype, "add").mockImplementation(function (
524 + this: Set<unknown>,
525 + value: unknown,
526 + ) {
527 + const result = add.call(this, value);
528 + if (value === task.id && !interrupted) {
529 + interrupted = true;
530 + Fiber.getCurrent()?.interruptUnsafe();
531 + }
532 + return result;
533 + });
534 + return yield* service
535 + .runNow({ id: task.id })
536 + .pipe(Effect.forkChild, Effect.flatMap(Fiber.await));
537 + });
538 + assert.isTrue(interrupted);
539 + assert.isTrue(Exit.hasInterrupts(first));
540 +
541 + const second = yield* service.runNow({ id: task.id });
542 + assert.equal(second.threadId, ThreadId.make("thread:launched"));
543 + assert.equal(yield* Ref.get(launches), 1);
544 + }).pipe(Effect.provide(layer));
545 + }).pipe(Effect.provide(SqlitePersistence.layerMemory)),
546 +);
apps/server/src/scheduledTasks/ScheduledTaskService.ts
+99 -91
@@ -786,20 +786,7 @@ export const layer = Layer.effect(
786 // Read before reserving: nothing between the reservation and its release may fail.
787 const startedAt = yield* localNow;
788 const startedAtIso = iso(startedAt);
789 - const reserved = yield* Ref.modify(activeRuns, (active) => {
790 - if (active.has(task.id)) return [false, active] as const;
791 - const next = new Set(active);
792 - next.add(task.id);
793 - return [true, next] as const;
794 - });
795 - if (!reserved) {
796 - if (trigger !== "scheduled") {
797 - return yield* taskError("Schedule task is already running.", { taskId: task.id });
798 - }
799 - return { task, threadId: null, dispatchFailed: false };
800 - }
801 -
802 - return yield* Effect.gen(function* () {
789 + const run = Effect.gen(function* () {
790 // Re-read stale poll snapshots so deleted, paused or postponed tasks cannot fire.
791 const active = yield* findTask(task.id);
792 if (active === null) {
@@ -1018,13 +1005,30 @@ export const layer = Layer.effect(
1005 Effect.onError((cause) =>
1006 releaseStuckRun(task, startedAtIso, errorMessage(cause), webhook?.deliveryId),
1007 ),
1021 - Effect.ensuring(
1022 - Ref.update(activeRuns, (active) => {
1023 - const next = new Set(active);
1024 - next.delete(task.id);
1025 - return next;
1026 - }),
1027 - ),
1008 + );
1009 +
1010 + // Reserve and release as one bracket so an interrupt cannot leave the task marked busy.
1011 + return yield* Effect.acquireUseRelease(
1012 + Ref.modify(activeRuns, (active) => {
1013 + if (active.has(task.id)) return [false, active] as const;
1014 + const next = new Set(active);
1015 + next.add(task.id);
1016 + return [true, next] as const;
1017 + }),
1018 + (reserved) =>
1019 + reserved
1020 + ? run
1021 + : trigger !== "scheduled"
1022 + ? Effect.fail(taskError("Schedule task is already running.", { taskId: task.id }))
1023 + : Effect.succeed({ task, threadId: null, dispatchFailed: false }),
1024 + (reserved) =>
1025 + reserved
1026 + ? Ref.update(activeRuns, (active) => {
1027 + const next = new Set(active);
1028 + next.delete(task.id);
1029 + return next;
1030 + })
1031 + : Effect.void,
1032 );
1033 });
1034
@@ -2243,83 +2247,87 @@ export const layer = Layer.effect(
2247 }
2248 yield* Effect.forEach(
2249 tasks,
2246 - ({ task_id: taskId }) =>
2247 - Effect.gen(function* () {
2248 - const reserved = yield* Ref.modify(pendingInFlight, (active) =>
2249 - active.has(taskId)
2250 - ? ([false, active] as const)
2251 - : ([true, new Set(active).add(taskId)] as const),
2252 - );
2253 - if (!reserved) return;
2254 - const permit = yield* webhookPermit(taskId);
2255 - // One worker per task waits on its permit, then drains persisted rows in order.
2256 - // Other tasks remain independent and ticks never reload an active task's payloads.
2257 - const dispatch = Effect.gen(function* () {
2258 - while (true) {
2259 - const [entry] = yield* sql<{
2260 - delivery_id: string;
2261 - task_json: string;
2262 - request_json: string;
2263 - state: string;
2264 - }>`
2250 + ({ task_id: taskId }) => {
2251 + // One worker per task waits on its permit, then drains persisted rows in order.
2252 + // Other tasks remain independent and ticks never reload an active task's payloads.
2253 + const dispatch = Effect.gen(function* () {
2254 + while (true) {
2255 + const [entry] = yield* sql<{
2256 + delivery_id: string;
2257 + task_json: string;
2258 + request_json: string;
2259 + state: string;
2260 + }>`
2261 SELECT delivery_id, task_json, request_json, state FROM scheduled_task_webhook_pending
2262 WHERE task_id = ${taskId} AND state IN ('dispatching', 'running', 'completed', 'failed') ORDER BY due_at, rowid LIMIT 1
2263 `.pipe(NodeSqliteClient.readOnly);
2268 - if (entry === undefined) return;
2269 - const remove = sql`DELETE FROM scheduled_task_webhook_pending WHERE delivery_id = ${entry.delivery_id}`;
2270 - if (entry.state === "completed" || entry.state === "failed") {
2271 - if (entry.state === "failed") {
2272 - yield* markDeliveryFailed(entry.delivery_id, "The run failed to start.");
2273 - yield* releaseFailedDelivery(entry.delivery_id);
2274 - }
2275 - yield* remove;
2276 - continue;
2277 - }
2278 - const prepared = yield* Effect.gen(function* () {
2279 - const task = yield* decodePendingTask(entry.task_json);
2280 - const request = yield* decodePendingRequest(entry.request_json);
2281 - const frameId = Buffer.from(yield* crypto.randomBytes(6)).toString("hex");
2282 - return { task, rendered: renderWebhookPrompt(task.prompt, request, frameId) };
2283 - }).pipe(Effect.exit);
2284 - if (Exit.isFailure(prepared)) {
2285 - if (Cause.hasInterruptsOnly(prepared.cause))
2286 - return yield* Effect.failCause(prepared.cause);
2287 - yield* markDeliveryFailed(
2288 - entry.delivery_id,
2289 - "The pending webhook payload could not be prepared.",
2290 - );
2291 - yield* releaseFailedDelivery(entry.delivery_id);
2292 - yield* remove;
2293 - continue;
2294 - }
2295 - const { task, rendered } = prepared.value;
2296 - if (rendered.prompt.trim().length > PROVIDER_SEND_TURN_MAX_INPUT_CHARS) {
2297 - yield* markDeliveryFailed(entry.delivery_id, PROMPT_TOO_LONG);
2264 + if (entry === undefined) return;
2265 + const remove = sql`DELETE FROM scheduled_task_webhook_pending WHERE delivery_id = ${entry.delivery_id}`;
2266 + if (entry.state === "completed" || entry.state === "failed") {
2267 + if (entry.state === "failed") {
2268 + yield* markDeliveryFailed(entry.delivery_id, "The run failed to start.");
2269 yield* releaseFailedDelivery(entry.delivery_id);
2299 - } else {
2300 - yield* sql`UPDATE scheduled_task_webhook_deliveries SET outcome = 'accepted', error = NULL, rendered_prompt = ${rendered.prompt.slice(0, WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT)}, missing_fields_json = ${encodeMissingFieldsJson(rendered.missing)} WHERE delivery_id = ${entry.delivery_id}`;
2301 - if (!(yield* runWebhookDelivery(task, entry.delivery_id, rendered.prompt)))
2302 - return;
2270 }
2271 yield* remove;
2272 + continue;
2273 }
2306 - });
2307 - yield* dispatch.pipe(
2308 - permit.withPermits(1),
2309 - Effect.catch((cause) =>
2310 - Effect.logWarning("Pending webhook dispatch failed", { cause }),
2311 - ),
2312 - Effect.ensuring(
2313 - Ref.update(pendingInFlight, (active) => {
2314 - const next = new Set(active);
2315 - next.delete(taskId);
2316 - return next;
2317 - }),
2318 - ),
2319 - Effect.withSpan("ScheduledTaskService.dispatchPendingWebhook", { root: true }),
2320 - Effect.forkIn(serviceScope),
2321 - );
2322 - }),
2274 + const prepared = yield* Effect.gen(function* () {
2275 + const task = yield* decodePendingTask(entry.task_json);
2276 + const request = yield* decodePendingRequest(entry.request_json);
2277 + const frameId = Buffer.from(yield* crypto.randomBytes(6)).toString("hex");
2278 + return { task, rendered: renderWebhookPrompt(task.prompt, request, frameId) };
2279 + }).pipe(Effect.exit);
2280 + if (Exit.isFailure(prepared)) {
2281 + if (Cause.hasInterruptsOnly(prepared.cause))
2282 + return yield* Effect.failCause(prepared.cause);
2283 + yield* markDeliveryFailed(
2284 + entry.delivery_id,
2285 + "The pending webhook payload could not be prepared.",
2286 + );
2287 + yield* releaseFailedDelivery(entry.delivery_id);
2288 + yield* remove;
2289 + continue;
2290 + }
2291 + const { task, rendered } = prepared.value;
2292 + if (rendered.prompt.trim().length > PROVIDER_SEND_TURN_MAX_INPUT_CHARS) {
2293 + yield* markDeliveryFailed(entry.delivery_id, PROMPT_TOO_LONG);
2294 + yield* releaseFailedDelivery(entry.delivery_id);
2295 + } else {
2296 + yield* sql`UPDATE scheduled_task_webhook_deliveries SET outcome = 'accepted', error = NULL, rendered_prompt = ${rendered.prompt.slice(0, WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT)}, missing_fields_json = ${encodeMissingFieldsJson(rendered.missing)} WHERE delivery_id = ${entry.delivery_id}`;
2297 + if (!(yield* runWebhookDelivery(task, entry.delivery_id, rendered.prompt))) return;
2298 + }
2299 + yield* remove;
2300 + }
2301 + });
2302 + // The worker owns its reservation, so an interrupt anywhere still releases it.
2303 + return Effect.acquireUseRelease(
2304 + Ref.modify(pendingInFlight, (active) =>
2305 + active.has(taskId)
2306 + ? ([false, active] as const)
2307 + : ([true, new Set(active).add(taskId)] as const),
2308 + ),
2309 + (reserved) =>
2310 + reserved
2311 + ? webhookPermit(taskId).pipe(
2312 + Effect.flatMap((permit) => dispatch.pipe(permit.withPermits(1))),
2313 + Effect.catch((cause) =>
2314 + Effect.logWarning("Pending webhook dispatch failed", { cause }),
2315 + ),
2316 + Effect.withSpan("ScheduledTaskService.dispatchPendingWebhook", {
2317 + root: true,
2318 + }),
2319 + )
2320 + : Effect.void,
2321 + (reserved) =>
2322 + reserved
2323 + ? Ref.update(pendingInFlight, (active) => {
2324 + const next = new Set(active);
2325 + next.delete(taskId);
2326 + return next;
2327 + })
2328 + : Effect.void,
2329 + ).pipe(Effect.forkIn(serviceScope));
2330 + },
2331 { discard: true },
2332 );
2333 });
apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts
+52
@@ -12,6 +12,7 @@ import * as DateTime from "effect/DateTime";
12 import * as Crypto from "effect/Crypto";
13 import * as Deferred from "effect/Deferred";
14 import * as Effect from "effect/Effect";
15 +import * as Exit from "effect/Exit";
16 import * as Fiber from "effect/Fiber";
17 import * as Layer from "effect/Layer";
18 import * as Queue from "effect/Queue";
@@ -1005,6 +1006,57 @@ it.effect("recovers a persisted coalescing window after restarting the service",
1006 );
1007 });
1008
1009 +it.effect("a pending dispatch interrupted at its permit does not block later polls", () => {
1010 + const workers = new Map<string, Effect.Effect<void>>();
1011 + // One shared dispatcher lets the test run the retry's forked worker to completion.
1012 + const mixed = new EffectScheduler.MixedScheduler("sync");
1013 + const dispatcher = mixed.makeDispatcher();
1014 + return withService(
1015 + ({ service, sql, launches }) =>
1016 + Effect.gen(function* () {
1017 + const { task } = yield* service.upsert(
1018 + yield* webhookTaskInput({
1019 + schedule: {
1020 + type: "webhook",
1021 + dedupe: { coalesceKey: "{{headers.content-type}}", coalesceWindowSeconds: 10 },
1022 + },
1023 + }),
1024 + );
1025 + yield* service.triggerWebhook(requestFor(task));
1026 + yield* TestClock.adjust("10 seconds");
1027 + const poll = workers.get("webhook-coalescing")!;
1028 +
1029 + const entered = yield* Deferred.make<Fiber.Fiber<unknown, unknown>>();
1030 + const interruptAtPermit = Tracer.make({
1031 + span: (options) => {
1032 + if (options.name === "ScheduledTaskService.webhookPermit") {
1033 + const fiber = Fiber.getCurrent()!;
1034 + Deferred.doneUnsafe(entered, Effect.succeed(fiber));
1035 + fiber.interruptUnsafe();
1036 + }
1037 + return new Tracer.NativeSpan(options);
1038 + },
1039 + });
1040 + const firstPoll = yield* poll.pipe(Effect.withTracer(interruptAtPermit), Effect.forkChild);
1041 + assert.isTrue(Exit.hasInterrupts(yield* Fiber.await(yield* Deferred.await(entered))));
1042 + yield* Fiber.await(firstPoll);
1043 +
1044 + yield* poll;
1045 + dispatcher.flush();
1046 +
1047 + assert.equal(yield* Queue.size(launches), 1);
1048 + assert.lengthOf(yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`, 0);
1049 + }),
1050 + { workers },
1051 + ).pipe(
1052 + Effect.provideService(EffectScheduler.Scheduler, {
1053 + executionMode: mixed.executionMode,
1054 + makeDispatcher: () => dispatcher,
1055 + shouldYield: (fiber) => mixed.shouldYield(fiber),
1056 + }),
1057 + );
1058 +});
1059 +
1060 it.effect("rolls back replay and pending state when recording a delivery fails", () => {
1061 const workers = new Map<string, Effect.Effect<void>>();
1062 return withService(