The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 19:45 UTC
fix(server): a missed fixed-time slot shows as failed on its scheduled task (#34815)
Julius Marminge committed
Nov 12, 2026 at 00:21 UTC
5c0eb5124e2eba13f240b18c4060342eb9fbc588
3 files changed
+333
-5
apps/server/src/scheduledTasks/ScheduledTaskService.schedule.test.ts
+164
@@ -392,3 +392,167 @@ it.effect(
392
}).pipe(Effect.provide(ScheduledTaskService.layer.pipe(Layer.provide(layerDependencies))));
393
}).pipe(Effect.provide(SqlitePersistence.layerMemory)),
394
);
395
+
396
+it.effect("a fixed-time slot missed while the server was off shows on the task", () =>
397
+ Effect.gen(function* () {
398
+ const at = (hour: number, minute = 0, day = 1) =>
399
+ DateTime.makeZonedUnsafe(
400
+ { year: 2026, month: 7, day, hour, minute, second: 0, millisecond: 0 },
401
+ { timeZone: DateTime.zoneMakeLocal(), adjustForTimeZone: true },
402
+ );
403
+ yield* TestClock.setTime(DateTime.toEpochMillis(at(8)));
404
+ const launches = yield* Ref.make(0);
405
+ const layerDependencies = Layer.mergeAll(
406
+ NodeCrypto.layer,
407
+ Scheduler.layer,
408
+ ManagedPolicy.layerTest(),
409
+ Layer.mock(ThreadLaunchService.ThreadLaunchService)({
410
+ launch: () =>
411
+ Ref.update(launches, (count) => count + 1).pipe(
412
+ Effect.as({ threadId: "thread-missed-slot", resumed: false } as never),
413
+ ),
414
+ }),
415
+ Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({
416
+ namedProjectsRoot: "/projects",
417
+ isScratchProject: () => Effect.succeed(false),
418
+ }),
419
+ Layer.mock(ThreadManagementService.ThreadManagementService)({}),
420
+ Layer.mock(SecretRequests.SecretRequests)({}),
421
+ );
422
+ const withServer = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
423
+ effect.pipe(
424
+ Effect.provide(ScheduledTaskService.layer.pipe(Layer.provide(layerDependencies))),
425
+ );
426
+
427
+ const taskId = yield* withServer(
428
+ Effect.gen(function* () {
429
+ const service = yield* ScheduledTaskService.ScheduledTaskService;
430
+ const { task } = yield* service.upsert(
431
+ yield* decodeUpsertInput({
432
+ title: "Morning bookkeeping",
433
+ prompt: "Update yesterday's notes.",
434
+ enabled: true,
435
+ schedule: { type: "fixed_time", timeOfDay: "09:00" },
436
+ projectId: "project-missed-slot",
437
+ workspaceStrategy: { type: "root" },
438
+ modelSelection: { instanceId: "codex", model: "gpt-5.4" },
439
+ runtimeMode: "full-access",
440
+ interactionMode: "default",
441
+ }),
442
+ );
443
+ yield* service.runNow({ id: task.id });
444
+ return task.id;
445
+ }),
446
+ );
447
+
448
+ // The server is off across the 09:00 slot and comes back at 09:30.
449
+ yield* TestClock.setTime(DateTime.toEpochMillis(at(9, 30)));
450
+ const [listed] = yield* withServer(
451
+ Effect.gen(function* () {
452
+ const service = yield* ScheduledTaskService.ScheduledTaskService;
453
+ yield* TestClock.adjust("6 seconds");
454
+ return (yield* service.list()).tasks;
455
+ }),
456
+ );
457
+
458
+ expect(listed?.id).toBe(taskId);
459
+ expect(yield* Ref.get(launches)).toBe(1);
460
+ expect(listed?.nextRunAt).toBe(DateTime.formatIso(DateTime.toUtc(at(9, 0, 2))));
461
+ // The skipped slot must not read as the earlier manual run's clean success.
462
+ expect({ status: listed?.lastRunStatus, error: listed?.lastRunError }).not.toEqual({
463
+ status: "succeeded",
464
+ error: null,
465
+ });
466
+ }).pipe(Effect.provide(SqlitePersistence.layerMemory)),
467
+);
468
+
469
+it.effect(
470
+ "a run made while the scheduler was busy is not overwritten by its stale missed slot",
471
+ () =>
472
+ Effect.gen(function* () {
473
+ const at = (hour: number, minute = 0, day = 1) =>
474
+ DateTime.makeZonedUnsafe(
475
+ { year: 2026, month: 7, day, hour, minute, second: 0, millisecond: 0 },
476
+ { timeZone: DateTime.zoneMakeLocal(), adjustForTimeZone: true },
477
+ );
478
+ yield* TestClock.setTime(DateTime.toEpochMillis(at(6)));
479
+ const slowStarted = yield* Deferred.make<void>();
480
+ const releaseSlow = yield* Deferred.make<void>();
481
+ const lastProcessed = yield* Deferred.make<void>();
482
+ const layerDependencies = Layer.mergeAll(
483
+ NodeCrypto.layer,
484
+ Scheduler.layer,
485
+ ManagedPolicy.layerTest(),
486
+ Layer.mock(ThreadLaunchService.ThreadLaunchService)({
487
+ launch: (input) =>
488
+ (input.title === "Slow"
489
+ ? Deferred.succeed(slowStarted, undefined).pipe(
490
+ Effect.andThen(Deferred.await(releaseSlow)),
491
+ )
492
+ : input.title === "Last"
493
+ ? Deferred.succeed(lastProcessed, undefined)
494
+ : Effect.void
495
+ ).pipe(Effect.as({ threadId: `thread-${input.title}`, resumed: false } as never)),
496
+ }),
497
+ Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({
498
+ namedProjectsRoot: "/projects",
499
+ isScratchProject: () => Effect.succeed(false),
500
+ }),
501
+ Layer.mock(ThreadManagementService.ThreadManagementService)({}),
502
+ Layer.mock(SecretRequests.SecretRequests)({}),
503
+ );
504
+ const withServer = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
505
+ effect.pipe(
506
+ Effect.provide(ScheduledTaskService.layer.pipe(Layer.provide(layerDependencies))),
507
+ );
508
+ const upsert = (
509
+ title: string,
510
+ schedule: { type: "interval"; everyMs: number } | { type: "fixed_time"; timeOfDay: string },
511
+ ) =>
512
+ Effect.gen(function* () {
513
+ const service = yield* ScheduledTaskService.ScheduledTaskService;
514
+ const { task } = yield* service.upsert(
515
+ yield* decodeUpsertInput({
516
+ title,
517
+ prompt: "Update the notes.",
518
+ enabled: true,
519
+ schedule,
520
+ projectId: "project-stale-slot",
521
+ workspaceStrategy: { type: "root" },
522
+ modelSelection: { instanceId: "codex", model: "gpt-5.4" },
523
+ runtimeMode: "full-access",
524
+ interactionMode: "default",
525
+ }),
526
+ );
527
+ return task.id;
528
+ });
529
+
530
+ // Due at 07:00, 08:00 and 08:30; the server is off until 09:30.
531
+ const morningId = yield* withServer(
532
+ Effect.gen(function* () {
533
+ yield* upsert("Slow", { type: "interval", everyMs: 60 * 60 * 1000 });
534
+ const id = yield* upsert("Morning", { type: "fixed_time", timeOfDay: "08:00" });
535
+ yield* upsert("Last", { type: "interval", everyMs: 150 * 60 * 1000 });
536
+ return id;
537
+ }),
538
+ );
539
+ yield* TestClock.setTime(DateTime.toEpochMillis(at(9, 30)));
540
+ const morning = yield* withServer(
541
+ Effect.gen(function* () {
542
+ const service = yield* ScheduledTaskService.ScheduledTaskService;
543
+ yield* TestClock.adjust("6 seconds");
544
+ yield* Deferred.await(slowStarted);
545
+ yield* service.runNow({ id: morningId });
546
+ yield* Deferred.succeed(releaseSlow, undefined);
547
+ yield* Deferred.await(lastProcessed);
548
+ return (yield* service.list()).tasks.find((task) => task.id === morningId);
549
+ }),
550
+ );
551
+
552
+ expect({ status: morning?.lastRunStatus, error: morning?.lastRunError }).toEqual({
553
+ status: "succeeded",
554
+ error: null,
555
+ });
556
+ expect(morning?.nextRunAt).toBe(DateTime.formatIso(DateTime.toUtc(at(8, 0, 2))));
557
+ }).pipe(Effect.provide(SqlitePersistence.layerMemory)),
558
+);
apps/server/src/scheduledTasks/ScheduledTaskService.test.ts
+89
@@ -17,6 +17,7 @@ import * as Effect from "effect/Effect";
17
import * as Exit from "effect/Exit";
18
import * as Layer from "effect/Layer";
19
import * as Logger from "effect/Logger";
20
+import * as Option from "effect/Option";
21
import * as SqlClient from "effect/sql/SqlClient";
22
23
import * as Fiber from "effect/Fiber";
@@ -418,6 +419,94 @@ it.effect("thread cleanup notifies subscribers and a finishing run cannot recrea
419
}).pipe(Effect.provide(SqlitePersistence.layerMemory)),
420
);
421
422
+it.effect.each([
423
+ { status: "failed", error: "Weekly usage limit reached" },
424
+ { status: "interrupted", error: "The run was stopped." },
425
+ { status: "cancelled", error: "The run was cancelled." },
426
+] as const)("a run that ends $status after its message was accepted records it", (outcome) =>
427
+ Effect.gen(function* () {
428
+ const projectId = ProjectId.make("project:test");
429
+ const ended = yield* Deferred.make<void>();
430
+ const sent = yield* Ref.make("");
431
+ const run = { id: "run:2", status: outcome.status, rootNodeId: "node:root" };
432
+ const layer = ScheduledTaskService.layer.pipe(
433
+ Layer.provide(
434
+ Layer.mergeAll(
435
+ ManagedPolicy.layerTest(),
436
+ Layer.mock(ThreadLaunchService.ThreadLaunchService)({}),
437
+ Layer.mock(ThreadManagementService.ThreadManagementService)({
438
+ sendToThread: (input) => Ref.set(sent, input.messageId).pipe(Effect.as({} as never)),
439
+ getThreadShell: () => Effect.succeed(threadShell(projectId)),
440
+ getProjectThreadRecords: () =>
441
+ Effect.map(
442
+ Ref.get(sent),
443
+ (userMessageId) =>
444
+ ({
445
+ runs: [
446
+ { id: "run:1", status: "completed", userMessageId: "message:earlier" },
447
+ { ...run, userMessageId },
448
+ ],
449
+ messages: [{ id: userMessageId, runId: "run:2" }],
450
+ turnItems: [
451
+ {
452
+ type: "error",
453
+ status: "failed",
454
+ runId: "run:2",
455
+ nodeId: "node:root",
456
+ failure: { message: "Weekly usage limit reached" },
457
+ },
458
+ ],
459
+ }) as never,
460
+ ),
461
+ waitForThread: (input) =>
462
+ Deferred.await(ended).pipe(
463
+ Effect.as({
464
+ threadId: input.threadId,
465
+ run: { ...run, id: input.runId, userMessageId: "" },
466
+ timedOut: false,
467
+ } as never),
468
+ ),
469
+ }),
470
+ Layer.mock(SecretRequests.SecretRequests)({}),
471
+ Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({
472
+ namedProjectsRoot: "/projects",
473
+ isScratchProject: () => Effect.succeed(false),
474
+ }),
475
+ NodeCrypto.layer,
476
+ Scheduler.layer,
477
+ ),
478
+ ),
479
+ );
480
+ yield* Effect.gen(function* () {
481
+ const service = yield* ScheduledTaskService.ScheduledTaskService;
482
+ const { task } = yield* service.upsert({
483
+ id: ScheduledTaskId.make("task:quota"),
484
+ title: "Quota bookkeeping",
485
+ prompt: "Update the notes",
486
+ enabled: true,
487
+ schedule: { type: "interval", everyMs: 60_000 },
488
+ projectId,
489
+ threadId: ThreadId.make("thread:quota"),
490
+ workspaceStrategy: { type: "root" },
491
+ modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5.4" },
492
+ runtimeMode: "full-access",
493
+ interactionMode: "default",
494
+ });
495
+ yield* service.runNow({ id: task.id });
496
+ const failed = yield* service.subscribeList().pipe(
497
+ Stream.map(({ tasks }) => tasks.find((candidate) => candidate.id === task.id)),
498
+ Stream.filter((current) => current?.lastRunStatus === "failed"),
499
+ Stream.runHead,
500
+ Effect.forkChild,
501
+ );
502
+ yield* Deferred.succeed(ended, undefined);
503
+ const recorded = Option.getOrUndefined(yield* Fiber.join(failed));
504
+ assert.equal(recorded?.lastRunError, outcome.error);
505
+ assert.equal(recorded?.runCount, 1);
506
+ }).pipe(Effect.provide(layer));
507
+ }).pipe(Effect.provide(SqlitePersistence.layerMemory)),
508
+);
509
+
510
it.effect("a bound run follows its thread when promotion moves it mid-send", () =>
511
Effect.gen(function* () {
512
const scratch = ProjectId.make("project:scratch");
apps/server/src/scheduledTasks/ScheduledTaskService.ts
+80
-5
@@ -57,6 +57,7 @@ import * as SecretRequests from "../secrets/SecretRequests.ts";
57
import * as ManagedPolicy from "../policy/ManagedPolicy.ts";
58
import * as DueHint from "../scheduling/DueHint.ts";
59
import * as Scheduler from "../scheduling/Scheduler.ts";
60
+import { latestRootProviderFailure } from "@t3tools/shared/orchestrationV2ThreadError";
61
import { isMissedFixedTimeRun, isSameSchedule, nextScheduledRunAt } from "./Schedule.ts";
62
import {
63
redactHeaders,
@@ -306,6 +307,13 @@ function iso(value: DateTime.DateTime): string {
307
return DateTime.formatIso(DateTime.toUtc(value));
308
}
309
310
+const RUN_OUTCOME_WAIT_MS = 24 * 60 * 60 * 1_000;
311
+const UNSUCCESSFUL_RUN_ERRORS = {
312
+ failed: "The run failed.",
313
+ interrupted: "The run was stopped.",
314
+ cancelled: "The run was cancelled.",
315
+ rolled_back: "The run was rolled back.",
316
+} as const;
317
const localNow = DateTime.withCurrentZoneLocal(DateTime.nowInCurrentZone);
318
319
function nextRunAt(
@@ -459,6 +467,7 @@ export const layer = Layer.effect(
467
const mac = yield* Mac.Mac;
468
const threadLaunch = yield* ThreadLaunchService.ThreadLaunchService;
469
const threadManagement = yield* ThreadManagementService.ThreadManagementService;
470
+ const serviceScope = yield* Effect.scope;
471
const managedFolders = yield* ManagedProjectFolders.ManagedProjectFolders;
472
const secretRequests = yield* SecretRequests.SecretRequests;
473
const scheduler = yield* Scheduler.Scheduler;
@@ -863,6 +872,52 @@ export const layer = Layer.effect(
872
),
873
);
874
875
+ // A dispatch is recorded once the message is accepted; the provider can still
876
+ // reject the turn after that, so follow the run and record its failure.
877
+ const recordRunFailure = Effect.fn("ScheduledTaskService.recordRunFailure")(function* (
878
+ taskId: ScheduledTaskId,
879
+ startedAtIso: string,
880
+ threadId: ThreadId,
881
+ messageId: MessageId,
882
+ ) {
883
+ const shell = yield* threadManagement.getThreadShell(threadId);
884
+ if (shell === null) return;
885
+ const thread = { projectId: shell.projectId, threadId };
886
+ const { runs } = yield* threadManagement.getProjectThreadRecords(thread, ["runs"]);
887
+ const dispatched = runs.find((run) => run.userMessageId === messageId);
888
+ if (dispatched === undefined) return;
889
+ const { run } = yield* threadManagement.waitForThread({
890
+ ...thread,
891
+ runId: dispatched.id,
892
+ timeoutMs: RUN_OUTCOME_WAIT_MS,
893
+ });
894
+ if (
895
+ run === null ||
896
+ run.status === "completed" ||
897
+ !ThreadManagementService.isTerminalRunStatus(run.status)
898
+ )
899
+ return;
900
+ const { turnItems, messages } = yield* threadManagement.getProjectThreadRecords(thread, [
901
+ "turnItems",
902
+ "messages",
903
+ ]);
904
+ // A queued message promoted to a steer cancels its run and joins the active one.
905
+ if (
906
+ run.status === "cancelled" &&
907
+ messages.find((message) => message.id === messageId)?.runId !== run.id
908
+ )
909
+ return;
910
+ const error =
911
+ latestRootProviderFailure(run, turnItems)?.message ?? UNSUCCESSFUL_RUN_ERRORS[run.status];
912
+ const updated = yield* sql`
913
+ UPDATE scheduled_tasks
914
+ SET last_run_status = 'failed', last_run_error = ${error}
915
+ WHERE task_id = ${taskId} AND last_run_at = ${startedAtIso} AND last_run_status = 'succeeded'
916
+ RETURNING task_id
917
+ `;
918
+ if (updated.length > 0) yield* notifyChanged;
919
+ });
920
+
921
const runTask = Effect.fn("ScheduledTaskService.runTask")(function* (
922
task: ScheduledTask,
923
trigger: "scheduled" | "manual" | "webhook",
@@ -1109,6 +1164,17 @@ export const layer = Layer.effect(
1164
),
1165
);
1166
if (written) yield* notifyChanged;
1167
+ if (written && threadId !== null) {
1168
+ yield* recordRunFailure(active.id, startedAtIso, threadId, messageId).pipe(
1169
+ Effect.catchCause((cause) =>
1170
+ Effect.logWarning("Could not record scheduled run outcome", {
1171
+ taskId: active.id,
1172
+ cause,
1173
+ }),
1174
+ ),
1175
+ Effect.forkIn(serviceScope),
1176
+ );
1177
+ }
1178
// The stored row can belong to a replacement, so report this dispatch separately.
1179
return { task: recorded, threadId, dispatchFailed: !runSucceeded };
1180
}).pipe(
@@ -1146,6 +1212,7 @@ export const layer = Layer.effect(
1212
task: ScheduledTask,
1213
now: DateTime.DateTime,
1214
reason = "missed",
1215
+ runError?: string,
1216
) {
1217
const next = nextRunAt(task, now);
1218
yield* Effect.logInfo("Skipping schedule task run", {
@@ -1154,17 +1221,22 @@ export const layer = Layer.effect(
1221
missedRunAt: task.nextRunAt,
1222
rescheduledTo: next,
1223
});
1157
- yield* sql`
1224
+ // `task` can be a stale batch snapshot; a run or edit since then already moved the row on.
1225
+ const updated = yield* sql`
1226
UPDATE scheduled_tasks
1227
SET next_run_at = ${next},
1228
updated_at = ${iso(now)}
1229
+ ${runError === undefined ? sql`` : sql`, last_run_status = 'failed', last_run_error = ${runError}`}
1230
WHERE task_id = ${task.id}
1231
+ AND next_run_at IS ${task.nextRunAt}
1232
+ AND last_run_at IS ${task.lastRunAt}
1233
+ RETURNING task_id
1234
`.pipe(
1235
Effect.mapError((cause) =>
1236
taskError("Could not reschedule missed schedule task run.", { taskId: task.id, cause }),
1237
),
1238
);
1167
- yield* notifyChanged;
1239
+ if (updated.length > 0) yield* notifyChanged;
1240
});
1241
1242
const runDueTasks = Effect.fn("ScheduledTaskService.runDueTasks")(function* () {
@@ -1202,7 +1274,12 @@ export const layer = Layer.effect(
1274
({ task, dueAt }) =>
1275
Effect.suspend(() =>
1276
isMissedFixedTimeRun(task.schedule, dueAt, now)
1205
- ? rescheduleMissedRun(task, now)
1277
+ ? rescheduleMissedRun(
1278
+ task,
1279
+ now,
1280
+ "missed",
1281
+ "Missed a scheduled run because the server was not running at that time.",
1282
+ )
1283
: runTask(task, "scheduled"),
1284
).pipe(
1285
Effect.catch((cause) =>
@@ -1915,8 +1992,6 @@ export const layer = Layer.effect(
1992
});
1993
});
1994
1918
- const serviceScope = yield* Effect.scope;
1919
-
1995
const releaseFailedDelivery = Effect.fn("ScheduledTaskService.releaseFailedDelivery")(
1996
function* (deliveryId: string) {
1997
yield* releaseReplay(deliveryId);