@devries-cameron20 / t3code-1 / commits / 5c0eb5124e
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): a missed fixed-time slot shows as failed on its scheduled task (#34815)

Julius Marminge committed Nov 12, 2026 at 00:21 UTC 5c0eb5124e2eba13f240b18c4060342eb9fbc588
3 files changed +333 -5
apps/server/src/scheduledTasks/ScheduledTaskService.schedule.test.ts
+164
@@ -392,3 +392,167 @@ it.effect(
392 }).pipe(Effect.provide(ScheduledTaskService.layer.pipe(Layer.provide(layerDependencies))));
393 }).pipe(Effect.provide(SqlitePersistence.layerMemory)),
394 );
395 +
396 +it.effect("a fixed-time slot missed while the server was off shows on the task", () =>
397 + Effect.gen(function* () {
398 + const at = (hour: number, minute = 0, day = 1) =>
399 + DateTime.makeZonedUnsafe(
400 + { year: 2026, month: 7, day, hour, minute, second: 0, millisecond: 0 },
401 + { timeZone: DateTime.zoneMakeLocal(), adjustForTimeZone: true },
402 + );
403 + yield* TestClock.setTime(DateTime.toEpochMillis(at(8)));
404 + const launches = yield* Ref.make(0);
405 + const layerDependencies = Layer.mergeAll(
406 + NodeCrypto.layer,
407 + Scheduler.layer,
408 + ManagedPolicy.layerTest(),
409 + Layer.mock(ThreadLaunchService.ThreadLaunchService)({
410 + launch: () =>
411 + Ref.update(launches, (count) => count + 1).pipe(
412 + Effect.as({ threadId: "thread-missed-slot", resumed: false } as never),
413 + ),
414 + }),
415 + Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({
416 + namedProjectsRoot: "/projects",
417 + isScratchProject: () => Effect.succeed(false),
418 + }),
419 + Layer.mock(ThreadManagementService.ThreadManagementService)({}),
420 + Layer.mock(SecretRequests.SecretRequests)({}),
421 + );
422 + const withServer = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
423 + effect.pipe(
424 + Effect.provide(ScheduledTaskService.layer.pipe(Layer.provide(layerDependencies))),
425 + );
426 +
427 + const taskId = yield* withServer(
428 + Effect.gen(function* () {
429 + const service = yield* ScheduledTaskService.ScheduledTaskService;
430 + const { task } = yield* service.upsert(
431 + yield* decodeUpsertInput({
432 + title: "Morning bookkeeping",
433 + prompt: "Update yesterday's notes.",
434 + enabled: true,
435 + schedule: { type: "fixed_time", timeOfDay: "09:00" },
436 + projectId: "project-missed-slot",
437 + workspaceStrategy: { type: "root" },
438 + modelSelection: { instanceId: "codex", model: "gpt-5.4" },
439 + runtimeMode: "full-access",
440 + interactionMode: "default",
441 + }),
442 + );
443 + yield* service.runNow({ id: task.id });
444 + return task.id;
445 + }),
446 + );
447 +
448 + // The server is off across the 09:00 slot and comes back at 09:30.
449 + yield* TestClock.setTime(DateTime.toEpochMillis(at(9, 30)));
450 + const [listed] = yield* withServer(
451 + Effect.gen(function* () {
452 + const service = yield* ScheduledTaskService.ScheduledTaskService;
453 + yield* TestClock.adjust("6 seconds");
454 + return (yield* service.list()).tasks;
455 + }),
456 + );
457 +
458 + expect(listed?.id).toBe(taskId);
459 + expect(yield* Ref.get(launches)).toBe(1);
460 + expect(listed?.nextRunAt).toBe(DateTime.formatIso(DateTime.toUtc(at(9, 0, 2))));
461 + // The skipped slot must not read as the earlier manual run's clean success.
462 + expect({ status: listed?.lastRunStatus, error: listed?.lastRunError }).not.toEqual({
463 + status: "succeeded",
464 + error: null,
465 + });
466 + }).pipe(Effect.provide(SqlitePersistence.layerMemory)),
467 +);
468 +
469 +it.effect(
470 + "a run made while the scheduler was busy is not overwritten by its stale missed slot",
471 + () =>
472 + Effect.gen(function* () {
473 + const at = (hour: number, minute = 0, day = 1) =>
474 + DateTime.makeZonedUnsafe(
475 + { year: 2026, month: 7, day, hour, minute, second: 0, millisecond: 0 },
476 + { timeZone: DateTime.zoneMakeLocal(), adjustForTimeZone: true },
477 + );
478 + yield* TestClock.setTime(DateTime.toEpochMillis(at(6)));
479 + const slowStarted = yield* Deferred.make<void>();
480 + const releaseSlow = yield* Deferred.make<void>();
481 + const lastProcessed = yield* Deferred.make<void>();
482 + const layerDependencies = Layer.mergeAll(
483 + NodeCrypto.layer,
484 + Scheduler.layer,
485 + ManagedPolicy.layerTest(),
486 + Layer.mock(ThreadLaunchService.ThreadLaunchService)({
487 + launch: (input) =>
488 + (input.title === "Slow"
489 + ? Deferred.succeed(slowStarted, undefined).pipe(
490 + Effect.andThen(Deferred.await(releaseSlow)),
491 + )
492 + : input.title === "Last"
493 + ? Deferred.succeed(lastProcessed, undefined)
494 + : Effect.void
495 + ).pipe(Effect.as({ threadId: `thread-${input.title}`, resumed: false } as never)),
496 + }),
497 + Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({
498 + namedProjectsRoot: "/projects",
499 + isScratchProject: () => Effect.succeed(false),
500 + }),
501 + Layer.mock(ThreadManagementService.ThreadManagementService)({}),
502 + Layer.mock(SecretRequests.SecretRequests)({}),
503 + );
504 + const withServer = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
505 + effect.pipe(
506 + Effect.provide(ScheduledTaskService.layer.pipe(Layer.provide(layerDependencies))),
507 + );
508 + const upsert = (
509 + title: string,
510 + schedule: { type: "interval"; everyMs: number } | { type: "fixed_time"; timeOfDay: string },
511 + ) =>
512 + Effect.gen(function* () {
513 + const service = yield* ScheduledTaskService.ScheduledTaskService;
514 + const { task } = yield* service.upsert(
515 + yield* decodeUpsertInput({
516 + title,
517 + prompt: "Update the notes.",
518 + enabled: true,
519 + schedule,
520 + projectId: "project-stale-slot",
521 + workspaceStrategy: { type: "root" },
522 + modelSelection: { instanceId: "codex", model: "gpt-5.4" },
523 + runtimeMode: "full-access",
524 + interactionMode: "default",
525 + }),
526 + );
527 + return task.id;
528 + });
529 +
530 + // Due at 07:00, 08:00 and 08:30; the server is off until 09:30.
531 + const morningId = yield* withServer(
532 + Effect.gen(function* () {
533 + yield* upsert("Slow", { type: "interval", everyMs: 60 * 60 * 1000 });
534 + const id = yield* upsert("Morning", { type: "fixed_time", timeOfDay: "08:00" });
535 + yield* upsert("Last", { type: "interval", everyMs: 150 * 60 * 1000 });
536 + return id;
537 + }),
538 + );
539 + yield* TestClock.setTime(DateTime.toEpochMillis(at(9, 30)));
540 + const morning = yield* withServer(
541 + Effect.gen(function* () {
542 + const service = yield* ScheduledTaskService.ScheduledTaskService;
543 + yield* TestClock.adjust("6 seconds");
544 + yield* Deferred.await(slowStarted);
545 + yield* service.runNow({ id: morningId });
546 + yield* Deferred.succeed(releaseSlow, undefined);
547 + yield* Deferred.await(lastProcessed);
548 + return (yield* service.list()).tasks.find((task) => task.id === morningId);
549 + }),
550 + );
551 +
552 + expect({ status: morning?.lastRunStatus, error: morning?.lastRunError }).toEqual({
553 + status: "succeeded",
554 + error: null,
555 + });
556 + expect(morning?.nextRunAt).toBe(DateTime.formatIso(DateTime.toUtc(at(8, 0, 2))));
557 + }).pipe(Effect.provide(SqlitePersistence.layerMemory)),
558 +);
apps/server/src/scheduledTasks/ScheduledTaskService.test.ts
+89
@@ -17,6 +17,7 @@ import * as Effect from "effect/Effect";
17 import * as Exit from "effect/Exit";
18 import * as Layer from "effect/Layer";
19 import * as Logger from "effect/Logger";
20 +import * as Option from "effect/Option";
21 import * as SqlClient from "effect/sql/SqlClient";
22
23 import * as Fiber from "effect/Fiber";
@@ -418,6 +419,94 @@ it.effect("thread cleanup notifies subscribers and a finishing run cannot recrea
419 }).pipe(Effect.provide(SqlitePersistence.layerMemory)),
420 );
421
422 +it.effect.each([
423 + { status: "failed", error: "Weekly usage limit reached" },
424 + { status: "interrupted", error: "The run was stopped." },
425 + { status: "cancelled", error: "The run was cancelled." },
426 +] as const)("a run that ends $status after its message was accepted records it", (outcome) =>
427 + Effect.gen(function* () {
428 + const projectId = ProjectId.make("project:test");
429 + const ended = yield* Deferred.make<void>();
430 + const sent = yield* Ref.make("");
431 + const run = { id: "run:2", status: outcome.status, rootNodeId: "node:root" };
432 + const layer = ScheduledTaskService.layer.pipe(
433 + Layer.provide(
434 + Layer.mergeAll(
435 + ManagedPolicy.layerTest(),
436 + Layer.mock(ThreadLaunchService.ThreadLaunchService)({}),
437 + Layer.mock(ThreadManagementService.ThreadManagementService)({
438 + sendToThread: (input) => Ref.set(sent, input.messageId).pipe(Effect.as({} as never)),
439 + getThreadShell: () => Effect.succeed(threadShell(projectId)),
440 + getProjectThreadRecords: () =>
441 + Effect.map(
442 + Ref.get(sent),
443 + (userMessageId) =>
444 + ({
445 + runs: [
446 + { id: "run:1", status: "completed", userMessageId: "message:earlier" },
447 + { ...run, userMessageId },
448 + ],
449 + messages: [{ id: userMessageId, runId: "run:2" }],
450 + turnItems: [
451 + {
452 + type: "error",
453 + status: "failed",
454 + runId: "run:2",
455 + nodeId: "node:root",
456 + failure: { message: "Weekly usage limit reached" },
457 + },
458 + ],
459 + }) as never,
460 + ),
461 + waitForThread: (input) =>
462 + Deferred.await(ended).pipe(
463 + Effect.as({
464 + threadId: input.threadId,
465 + run: { ...run, id: input.runId, userMessageId: "" },
466 + timedOut: false,
467 + } as never),
468 + ),
469 + }),
470 + Layer.mock(SecretRequests.SecretRequests)({}),
471 + Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({
472 + namedProjectsRoot: "/projects",
473 + isScratchProject: () => Effect.succeed(false),
474 + }),
475 + NodeCrypto.layer,
476 + Scheduler.layer,
477 + ),
478 + ),
479 + );
480 + yield* Effect.gen(function* () {
481 + const service = yield* ScheduledTaskService.ScheduledTaskService;
482 + const { task } = yield* service.upsert({
483 + id: ScheduledTaskId.make("task:quota"),
484 + title: "Quota bookkeeping",
485 + prompt: "Update the notes",
486 + enabled: true,
487 + schedule: { type: "interval", everyMs: 60_000 },
488 + projectId,
489 + threadId: ThreadId.make("thread:quota"),
490 + workspaceStrategy: { type: "root" },
491 + modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" },
492 + runtimeMode: "full-access",
493 + interactionMode: "default",
494 + });
495 + yield* service.runNow({ id: task.id });
496 + const failed = yield* service.subscribeList().pipe(
497 + Stream.map(({ tasks }) => tasks.find((candidate) => candidate.id === task.id)),
498 + Stream.filter((current) => current?.lastRunStatus === "failed"),
499 + Stream.runHead,
500 + Effect.forkChild,
501 + );
502 + yield* Deferred.succeed(ended, undefined);
503 + const recorded = Option.getOrUndefined(yield* Fiber.join(failed));
504 + assert.equal(recorded?.lastRunError, outcome.error);
505 + assert.equal(recorded?.runCount, 1);
506 + }).pipe(Effect.provide(layer));
507 + }).pipe(Effect.provide(SqlitePersistence.layerMemory)),
508 +);
509 +
510 it.effect("a bound run follows its thread when promotion moves it mid-send", () =>
511 Effect.gen(function* () {
512 const scratch = ProjectId.make("project:scratch");
apps/server/src/scheduledTasks/ScheduledTaskService.ts
+80 -5
@@ -57,6 +57,7 @@ import * as SecretRequests from "../secrets/SecretRequests.ts";
57 import * as ManagedPolicy from "../policy/ManagedPolicy.ts";
58 import * as DueHint from "../scheduling/DueHint.ts";
59 import * as Scheduler from "../scheduling/Scheduler.ts";
60 +import { latestRootProviderFailure } from "@t3tools/shared/orchestrationV2ThreadError";
61 import { isMissedFixedTimeRun, isSameSchedule, nextScheduledRunAt } from "./Schedule.ts";
62 import {
63 redactHeaders,
@@ -306,6 +307,13 @@ function iso(value: DateTime.DateTime): string {
307 return DateTime.formatIso(DateTime.toUtc(value));
308 }
309
310 +const RUN_OUTCOME_WAIT_MS = 24 * 60 * 60 * 1_000;
311 +const UNSUCCESSFUL_RUN_ERRORS = {
312 + failed: "The run failed.",
313 + interrupted: "The run was stopped.",
314 + cancelled: "The run was cancelled.",
315 + rolled_back: "The run was rolled back.",
316 +} as const;
317 const localNow = DateTime.withCurrentZoneLocal(DateTime.nowInCurrentZone);
318
319 function nextRunAt(
@@ -459,6 +467,7 @@ export const layer = Layer.effect(
467 const mac = yield* Mac.Mac;
468 const threadLaunch = yield* ThreadLaunchService.ThreadLaunchService;
469 const threadManagement = yield* ThreadManagementService.ThreadManagementService;
470 + const serviceScope = yield* Effect.scope;
471 const managedFolders = yield* ManagedProjectFolders.ManagedProjectFolders;
472 const secretRequests = yield* SecretRequests.SecretRequests;
473 const scheduler = yield* Scheduler.Scheduler;
@@ -863,6 +872,52 @@ export const layer = Layer.effect(
872 ),
873 );
874
875 + // A dispatch is recorded once the message is accepted; the provider can still
876 + // reject the turn after that, so follow the run and record its failure.
877 + const recordRunFailure = Effect.fn("ScheduledTaskService.recordRunFailure")(function* (
878 + taskId: ScheduledTaskId,
879 + startedAtIso: string,
880 + threadId: ThreadId,
881 + messageId: MessageId,
882 + ) {
883 + const shell = yield* threadManagement.getThreadShell(threadId);
884 + if (shell === null) return;
885 + const thread = { projectId: shell.projectId, threadId };
886 + const { runs } = yield* threadManagement.getProjectThreadRecords(thread, ["runs"]);
887 + const dispatched = runs.find((run) => run.userMessageId === messageId);
888 + if (dispatched === undefined) return;
889 + const { run } = yield* threadManagement.waitForThread({
890 + ...thread,
891 + runId: dispatched.id,
892 + timeoutMs: RUN_OUTCOME_WAIT_MS,
893 + });
894 + if (
895 + run === null ||
896 + run.status === "completed" ||
897 + !ThreadManagementService.isTerminalRunStatus(run.status)
898 + )
899 + return;
900 + const { turnItems, messages } = yield* threadManagement.getProjectThreadRecords(thread, [
901 + "turnItems",
902 + "messages",
903 + ]);
904 + // A queued message promoted to a steer cancels its run and joins the active one.
905 + if (
906 + run.status === "cancelled" &&
907 + messages.find((message) => message.id === messageId)?.runId !== run.id
908 + )
909 + return;
910 + const error =
911 + latestRootProviderFailure(run, turnItems)?.message ?? UNSUCCESSFUL_RUN_ERRORS[run.status];
912 + const updated = yield* sql`
913 + UPDATE scheduled_tasks
914 + SET last_run_status = 'failed', last_run_error = ${error}
915 + WHERE task_id = ${taskId} AND last_run_at = ${startedAtIso} AND last_run_status = 'succeeded'
916 + RETURNING task_id
917 + `;
918 + if (updated.length > 0) yield* notifyChanged;
919 + });
920 +
921 const runTask = Effect.fn("ScheduledTaskService.runTask")(function* (
922 task: ScheduledTask,
923 trigger: "scheduled" | "manual" | "webhook",
@@ -1109,6 +1164,17 @@ export const layer = Layer.effect(
1164 ),
1165 );
1166 if (written) yield* notifyChanged;
1167 + if (written && threadId !== null) {
1168 + yield* recordRunFailure(active.id, startedAtIso, threadId, messageId).pipe(
1169 + Effect.catchCause((cause) =>
1170 + Effect.logWarning("Could not record scheduled run outcome", {
1171 + taskId: active.id,
1172 + cause,
1173 + }),
1174 + ),
1175 + Effect.forkIn(serviceScope),
1176 + );
1177 + }
1178 // The stored row can belong to a replacement, so report this dispatch separately.
1179 return { task: recorded, threadId, dispatchFailed: !runSucceeded };
1180 }).pipe(
@@ -1146,6 +1212,7 @@ export const layer = Layer.effect(
1212 task: ScheduledTask,
1213 now: DateTime.DateTime,
1214 reason = "missed",
1215 + runError?: string,
1216 ) {
1217 const next = nextRunAt(task, now);
1218 yield* Effect.logInfo("Skipping schedule task run", {
@@ -1154,17 +1221,22 @@ export const layer = Layer.effect(
1221 missedRunAt: task.nextRunAt,
1222 rescheduledTo: next,
1223 });
1157 - yield* sql`
1224 + // `task` can be a stale batch snapshot; a run or edit since then already moved the row on.
1225 + const updated = yield* sql`
1226 UPDATE scheduled_tasks
1227 SET next_run_at = ${next},
1228 updated_at = ${iso(now)}
1229 + ${runError === undefined ? sql`` : sql`, last_run_status = 'failed', last_run_error = ${runError}`}
1230 WHERE task_id = ${task.id}
1231 + AND next_run_at IS ${task.nextRunAt}
1232 + AND last_run_at IS ${task.lastRunAt}
1233 + RETURNING task_id
1234 `.pipe(
1235 Effect.mapError((cause) =>
1236 taskError("Could not reschedule missed schedule task run.", { taskId: task.id, cause }),
1237 ),
1238 );
1167 - yield* notifyChanged;
1239 + if (updated.length > 0) yield* notifyChanged;
1240 });
1241
1242 const runDueTasks = Effect.fn("ScheduledTaskService.runDueTasks")(function* () {
@@ -1202,7 +1274,12 @@ export const layer = Layer.effect(
1274 ({ task, dueAt }) =>
1275 Effect.suspend(() =>
1276 isMissedFixedTimeRun(task.schedule, dueAt, now)
1205 - ? rescheduleMissedRun(task, now)
1277 + ? rescheduleMissedRun(
1278 + task,
1279 + now,
1280 + "missed",
1281 + "Missed a scheduled run because the server was not running at that time.",
1282 + )
1283 : runTask(task, "scheduled"),
1284 ).pipe(
1285 Effect.catch((cause) =>
@@ -1915,8 +1992,6 @@ export const layer = Layer.effect(
1992 });
1993 });
1994
1918 - const serviceScope = yield* Effect.scope;
1919 -
1995 const releaseFailedDelivery = Effect.fn("ScheduledTaskService.releaseFailedDelivery")(
1996 function* (deliveryId: string) {
1997 yield* releaseReplay(deliveryId);