@devries-cameron20 / t3code Mirror · read-only / commits / f2aa99694b
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

fix(server): report held recurring task occurrences (#38906)

Co-authored-by: Julius Marminge <julius0216@outlook.com>

Wu Shuwen committed Nov 19, 2026 at 16:09 UTC f2aa99694b9e65b146dedeb4660f5ff00adce6c5
4 files changed +177 -18
apps/server/src/scheduledTasks/ScheduledTaskService.schedule.test.ts
+47 -2
@@ -14,6 +14,7 @@ import * as Effect from "effect/Effect";
14 import * as Layer from "effect/Layer";
15 import * as Ref from "effect/Ref";
16 import * as Schema from "effect/Schema";
17 +import * as SqlClient from "effect/sql/SqlClient";
18 import * as TestClock from "effect/testing/TestClock";
19
20 import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts";
@@ -195,6 +196,8 @@ it.effect("keeps one queued occurrence of a recurring task while its bound threa
196 readonly messages: ReadonlyArray<OrchestrationV2ConversationMessage>;
197 }>({ runs: [activeRun], messages: [] });
198 const runDueWork = yield* Ref.make<Effect.Effect<void>>(Effect.void);
199 + const sql = yield* SqlClient.SqlClient;
200 + const outcomeDuringLookup = yield* Ref.make(false);
201 const layerDependencies = Layer.mergeAll(
202 NodeCrypto.layer,
203 ManagedPolicy.layerTest(),
@@ -241,7 +244,18 @@ it.effect("keeps one queued occurrence of a recurring task while its bound threa
244 { runs: [...current.runs, run], messages: [...current.messages, message] },
245 ];
246 }),
244 - getThreadRecords: () => Ref.get(thread).pipe(Effect.map((records) => records as never)),
247 + getThreadRecords: (_id, fields) =>
248 + Effect.gen(function* () {
249 + if (
250 + fields.some((field) => field === "messages") &&
251 + (yield* Ref.get(outcomeDuringLookup))
252 + ) {
253 + yield* Ref.set(outcomeDuringLookup, false);
254 + yield* sql`UPDATE scheduled_tasks SET last_run_status = 'failed',
255 + last_run_error = 'Provider failed' WHERE task_id = 'scheduled-task:busy-thread'`;
256 + }
257 + return (yield* Ref.get(thread)) as never;
258 + }).pipe(Effect.orDie),
259 getThreadShell: () =>
260 Effect.succeed({
261 projectId: "project-busy-thread",
@@ -292,6 +306,34 @@ it.effect("keeps one queued occurrence of a recurring task while its bound threa
306 Date.parse("2026-07-01T09:03:03.000Z"),
307 );
308
309 + yield* Ref.update(thread, (current) => ({
310 + ...current,
311 + runs: current.runs.map((run) =>
312 + run.status === "queued" ? { ...run, queueHeld: true } : run,
313 + ),
314 + }));
315 + yield* TestClock.adjust("61 seconds");
316 + yield* Effect.flatten(Ref.get(runDueWork));
317 + const [blocked] = (yield* service.list()).tasks;
318 + expect(blocked?.lastRunStatus).toBe("failed");
319 + expect(blocked?.lastRunError).toBe(
320 + "A previous occurrence is held in the thread queue. Resume or remove it to run this task again.",
321 + );
322 + expect(yield* queuedOccurrences).toBe(1);
323 +
324 + // The prior run can finish while the poll is looking up its held occurrence.
325 + yield* sql`UPDATE scheduled_tasks SET last_run_status = 'succeeded', last_run_error = NULL
326 + WHERE task_id = ${task.id}`;
327 + yield* Ref.set(outcomeDuringLookup, true);
328 + yield* TestClock.adjust("61 seconds");
329 + yield* Effect.flatten(Ref.get(runDueWork));
330 + const [raced] = (yield* service.list()).tasks;
331 + expect(raced?.lastRunStatus).toBe("failed");
332 + expect(raced?.lastRunError).toBe("Provider failed");
333 + expect(Date.parse(raced?.nextRunAt ?? "")).toBeGreaterThan(
334 + Date.parse(blocked?.nextRunAt ?? ""),
335 + );
336 +
337 // Run now is an explicit request, so it still queues behind the held one.
338 yield* service.runNow({ id: task.id });
339 expect(yield* queuedOccurrences).toBe(2);
@@ -306,6 +348,9 @@ it.effect("keeps one queued occurrence of a recurring task while its bound threa
348 yield* TestClock.adjust("61 seconds");
349 yield* Effect.flatten(Ref.get(runDueWork));
350 expect(yield* queuedOccurrences).toBe(1);
351 + const [resumed] = (yield* service.list()).tasks;
352 + expect(resumed?.lastRunStatus).toBe("succeeded");
353 + expect(resumed?.lastRunError).toBeNull();
354
355 // A one-time task's only scheduled occurrence still sends behind a queued Run now.
356 const { task: once } = yield* service.upsert(
@@ -314,7 +359,7 @@ it.effect("keeps one queued occurrence of a recurring task while its bound threa
359 title: "Check once",
360 prompt: "Check once",
361 enabled: true,
317 - schedule: { type: "once", runAt: "2026-07-01T09:05:00.000Z" },
362 + schedule: { type: "once", runAt: "2026-07-01T09:07:00.000Z" },
363 projectId: "project-busy-thread",
364 threadId: "thread:busy",
365 workspaceStrategy: { type: "root" },
apps/server/src/scheduledTasks/ScheduledTaskService.test.ts
+11 -4
@@ -422,9 +422,10 @@ it.effect("thread cleanup notifies subscribers and a finishing run cannot recrea
422 );
423
424 it.effect.each([
425 - { status: "failed", error: "Weekly usage limit reached" },
426 - { status: "interrupted", error: "The run was stopped." },
427 - { status: "cancelled", error: "The run was cancelled." },
425 + { status: "failed", error: "Weekly usage limit reached", heldWarning: false },
426 + { status: "failed", error: "Weekly usage limit reached", heldWarning: true },
427 + { status: "interrupted", error: "The run was stopped.", heldWarning: false },
428 + { status: "cancelled", error: "The run was cancelled.", heldWarning: false },
429 ] as const)("a run that ends $status after its message was accepted records it", (outcome) =>
430 Effect.gen(function* () {
431 const projectId = ProjectId.make("project:test");
@@ -495,9 +496,15 @@ it.effect.each([
496 interactionMode: "default",
497 });
498 yield* service.runNow({ id: task.id });
499 + if (outcome.heldWarning) {
500 + const sql = yield* SqlClient.SqlClient;
501 + yield* sql`UPDATE scheduled_tasks SET last_run_status = 'failed',
502 + last_run_error = 'A previous occurrence is held in the thread queue. Resume or remove it to run this task again.'
503 + WHERE task_id = ${task.id}`;
504 + }
505 const failed = yield* service.subscribeList().pipe(
506 Stream.map(({ tasks }) => tasks.find((candidate) => candidate.id === task.id)),
500 - Stream.filter((current) => current?.lastRunStatus === "failed"),
507 + Stream.filter((current) => current?.lastRunError === outcome.error),
508 Stream.runHead,
509 Effect.forkChild,
510 );
apps/server/src/scheduledTasks/ScheduledTaskService.ts
+41 -12
@@ -311,6 +311,8 @@ function iso(value: DateTime.DateTime): string {
311 }
312
313 const RUN_OUTCOME_WAIT_MS = 24 * 60 * 60 * 1_000;
314 +const HELD_OCCURRENCE_ERROR =
315 + "A previous occurrence is held in the thread queue. Resume or remove it to run this task again.";
316 const UNSUCCESSFUL_RUN_ERRORS = {
317 failed: "The run failed.",
318 interrupted: "The run was stopped.",
@@ -848,20 +850,28 @@ export const layer = Layer.effect(
850
851 // Reads every run row but only the queued runs' messages, so the cost per
852 // due interval tracks the thread's run count, not its transcript.
851 - const hasQueuedOccurrence = Effect.fn("ScheduledTaskService.hasQueuedOccurrence")(function* (
853 + const queuedOccurrence = Effect.fn("ScheduledTaskService.queuedOccurrence")(function* (
854 taskId: ScheduledTaskId,
855 threadId: ThreadId,
856 ) {
857 const { runs } = yield* threadManagement.getThreadRecords(threadId, ["runs"]);
858 const queued = runs.filter((run) => run.status === "queued");
857 - if (queued.length === 0) return false;
859 + if (queued.length === 0) return null;
860 const queuedMessageIds = new Set(queued.map((run) => run.userMessageId));
861 const { messages } = yield* threadManagement.getThreadRecords(threadId, ["messages"], {
862 messageIds: [...queuedMessageIds],
863 });
862 - return messages.some(
863 - (message) => queuedMessageIds.has(message.id) && message.scheduledTaskId === taskId,
864 + const taskMessageIds = new Set(
865 + messages
866 + .filter((message) => message.scheduledTaskId === taskId)
867 + .map((message) => message.id),
868 );
869 + const occurrences = queued.filter((run) => taskMessageIds.has(run.userMessageId));
870 + return occurrences.some((run) => run.queueHeld === true)
871 + ? "held"
872 + : occurrences.length > 0
873 + ? "queued"
874 + : null;
875 });
876
877 // The run already started the thread, so a failed write only costs the binding.
@@ -940,7 +950,9 @@ export const layer = Layer.effect(
950 const updated = yield* sql`
951 UPDATE scheduled_tasks
952 SET last_run_status = 'failed', last_run_error = ${error}
943 - WHERE task_id = ${taskId} AND last_run_at = ${startedAtIso} AND last_run_status = 'succeeded'
953 + WHERE task_id = ${taskId} AND last_run_at = ${startedAtIso}
954 + AND (last_run_status = 'succeeded'
955 + OR (last_run_status = 'failed' AND last_run_error = ${HELD_OCCURRENCE_ERROR}))
956 RETURNING task_id
957 `;
958 if (updated.length > 0) yield* notifyChanged;
@@ -998,13 +1010,21 @@ export const layer = Layer.effect(
1010 if (
1011 trigger === "scheduled" &&
1012 active.schedule.type !== "once" &&
1001 - active.threadId !== null &&
1002 - (yield* hasQueuedOccurrence(active.id, ThreadId.make(active.threadId)).pipe(
1003 - Effect.orElseSucceed(() => false),
1004 - ))
1013 + active.threadId !== null
1014 ) {
1006 - yield* rescheduleMissedRun(active, startedAt, "previous occurrence still queued");
1007 - return { task: active, threadId: null, dispatchFailed: false };
1015 + const occurrence = yield* queuedOccurrence(
1016 + active.id,
1017 + ThreadId.make(active.threadId),
1018 + ).pipe(Effect.orElseSucceed(() => null));
1019 + if (occurrence !== null) {
1020 + yield* rescheduleMissedRun(
1021 + active,
1022 + startedAt,
1023 + "previous occurrence still queued",
1024 + occurrence === "held" ? HELD_OCCURRENCE_ERROR : undefined,
1025 + );
1026 + return { task: active, threadId: null, dispatchFailed: false };
1027 + }
1028 }
1029 // Reject queued deliveries after pause, same-id recreation or trigger changes.
1030 if (webhook !== undefined) {
@@ -1254,7 +1274,16 @@ export const layer = Layer.effect(
1274 UPDATE scheduled_tasks
1275 SET next_run_at = ${next},
1276 updated_at = ${iso(now)}
1257 - ${runError === undefined ? sql`` : sql`, last_run_status = 'failed', last_run_error = ${runError}`}
1277 + ${
1278 + runError === undefined
1279 + ? sql``
1280 + : sql`, last_run_status = CASE
1281 + WHEN last_run_status IS ${task.lastRunStatus} AND last_run_error IS ${task.lastRunError}
1282 + THEN 'failed' ELSE last_run_status END,
1283 + last_run_error = CASE
1284 + WHEN last_run_status IS ${task.lastRunStatus} AND last_run_error IS ${task.lastRunError}
1285 + THEN ${runError} ELSE last_run_error END`
1286 + }
1287 WHERE task_id = ${task.id}
1288 AND next_run_at IS ${task.nextRunAt}
1289 AND last_run_at IS ${task.lastRunAt}
scripts/scenarios/held-scheduled-occurrence.e2e.ts new
+78
@@ -0,0 +1,78 @@
1 +import { expect } from "e2e";
2 +import * as Effect from "effect/Effect";
3 +import * as FileSystem from "effect/FileSystem";
4 +import * as Path from "effect/Path";
5 +import * as Schema from "effect/Schema";
6 +
7 +import { T3_HOME } from "../e2e.config.ts";
8 +import { runPromise, test, WEB } from "./t3.ts";
9 +
10 +const SettingsJson = Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown));
11 +const decodeSettings = Schema.decodeEffect(SettingsJson);
12 +const encodeSettings = Schema.encodeEffect(SettingsJson);
13 +
14 +const threadRuntimeSettings = Effect.gen(function* () {
15 + const fs = yield* FileSystem.FileSystem;
16 + const settingsPath = (yield* Path.Path).join(T3_HOME, "userdata/settings.json");
17 + const original = yield* fs.readFileString(settingsPath);
18 + const settings = yield* decodeSettings(original);
19 + return {
20 + supervise: fs.writeFileString(
21 + settingsPath,
22 + yield* encodeSettings({ ...settings, defaultRuntimeMode: "approval-required" }),
23 + ),
24 + restore: fs.writeFileString(settingsPath, original),
25 + };
26 +});
27 +
28 +test(
29 + "A recurring task reports when its queued occurrence is held",
30 + WEB,
31 + async ({ t3, app, screen }) => {
32 + const settings = await runPromise(threadRuntimeSettings);
33 + try {
34 + // Supervised mode makes the approval recording wait for an answer, which holds its turn open.
35 + await runPromise(settings.supervise);
36 + const project = await t3.project("scheduled-hold");
37 + await t3.thread({ project, title: "Earlier work", messages: ["Start"] });
38 + const thread = await t3.thread({
39 + project,
40 + title: "Waiting scheduled thread",
41 + messages: ["Ask a question"],
42 + model: "approval",
43 + });
44 +
45 + await t3.open("/settings/scheduled-tasks");
46 + await screen.getByRole("button", "New task").tap();
47 + await screen.getByRole("textbox", "Name").fill("Held recurring task");
48 + await screen.getByRole("combobox", "Destination").tap();
49 + await screen.getByRole("option", "One thread, continuing the conversation").tap();
50 + await screen.getByRole("combobox", "Destination thread").tap();
51 + await screen.getByRole("option", "Waiting scheduled thread").tap();
52 + await screen.getByRole("textbox", "Prompt").fill("Scheduled check prompt");
53 + await screen.getByRole("button", "Every interval").tap();
54 + await screen.getByRole("spinbutton", "Run every").fill("1");
55 + await screen.getByRole("button", "Create task").tap();
56 +
57 + await t3.open(thread);
58 + await screen.getByRole("button", "Approve", { exact: true }).waitFor();
59 + await screen
60 + .getByRole("button", "Automation Held recurring task")
61 + .waitFor({ timeout: 100_000 });
62 + // Stopping the active turn holds the queued scheduled occurrence.
63 + await screen.getByRole("button", "Stop generation").tap();
64 + await screen.getByRole("button", "Resume thread").waitFor();
65 + await screen.getByRole("button", "Edit queued message").waitFor();
66 +
67 + await app.open("/settings/scheduled-tasks");
68 + await expect(
69 + screen.getByText(
70 + "A previous occurrence is held in the thread queue. Resume or remove it to run this task again.",
71 + ),
72 + ).toBeVisible({ timeout: 100_000 });
73 + await app.screenshot("held-scheduled-task");
74 + } finally {
75 + await runPromise(settings.restore);
76 + }
77 + },
78 +);