The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 06:00 UTC
fix(server): report held recurring task occurrences (#38906)
Co-authored-by: Julius Marminge <julius0216@outlook.com>
Wu Shuwen committed
Nov 19, 2026 at 16:09 UTC
f2aa99694b9e65b146dedeb4660f5ff00adce6c5
4 files changed
+177
-18
apps/server/src/scheduledTasks/ScheduledTaskService.schedule.test.ts
+47
-2
@@ -14,6 +14,7 @@ import * as Effect from "effect/Effect";
14
import * as Layer from "effect/Layer";
15
import * as Ref from "effect/Ref";
16
import * as Schema from "effect/Schema";
17
+import * as SqlClient from "effect/sql/SqlClient";
18
import * as TestClock from "effect/testing/TestClock";
19
20
import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts";
@@ -195,6 +196,8 @@ it.effect("keeps one queued occurrence of a recurring task while its bound threa
196
readonly messages: ReadonlyArray<OrchestrationV2ConversationMessage>;
197
}>({ runs: [activeRun], messages: [] });
198
const runDueWork = yield* Ref.make<Effect.Effect<void>>(Effect.void);
199
+ const sql = yield* SqlClient.SqlClient;
200
+ const outcomeDuringLookup = yield* Ref.make(false);
201
const layerDependencies = Layer.mergeAll(
202
NodeCrypto.layer,
203
ManagedPolicy.layerTest(),
@@ -241,7 +244,18 @@ it.effect("keeps one queued occurrence of a recurring task while its bound threa
244
{ runs: [...current.runs, run], messages: [...current.messages, message] },
245
];
246
}),
244
- getThreadRecords: () => Ref.get(thread).pipe(Effect.map((records) => records as never)),
247
+ getThreadRecords: (_id, fields) =>
248
+ Effect.gen(function* () {
249
+ if (
250
+ fields.some((field) => field === "messages") &&
251
+ (yield* Ref.get(outcomeDuringLookup))
252
+ ) {
253
+ yield* Ref.set(outcomeDuringLookup, false);
254
+ yield* sql`UPDATE scheduled_tasks SET last_run_status = 'failed',
255
+ last_run_error = 'Provider failed' WHERE task_id = 'scheduled-task:busy-thread'`;
256
+ }
257
+ return (yield* Ref.get(thread)) as never;
258
+ }).pipe(Effect.orDie),
259
getThreadShell: () =>
260
Effect.succeed({
261
projectId: "project-busy-thread",
@@ -292,6 +306,34 @@ it.effect("keeps one queued occurrence of a recurring task while its bound threa
306
Date.parse("2026-07-01T09:03:03.000Z"),
307
);
308
309
+ yield* Ref.update(thread, (current) => ({
310
+ ...current,
311
+ runs: current.runs.map((run) =>
312
+ run.status === "queued" ? { ...run, queueHeld: true } : run,
313
+ ),
314
+ }));
315
+ yield* TestClock.adjust("61 seconds");
316
+ yield* Effect.flatten(Ref.get(runDueWork));
317
+ const [blocked] = (yield* service.list()).tasks;
318
+ expect(blocked?.lastRunStatus).toBe("failed");
319
+ expect(blocked?.lastRunError).toBe(
320
+ "A previous occurrence is held in the thread queue. Resume or remove it to run this task again.",
321
+ );
322
+ expect(yield* queuedOccurrences).toBe(1);
323
+
324
+ // The prior run can finish while the poll is looking up its held occurrence.
325
+ yield* sql`UPDATE scheduled_tasks SET last_run_status = 'succeeded', last_run_error = NULL
326
+ WHERE task_id = ${task.id}`;
327
+ yield* Ref.set(outcomeDuringLookup, true);
328
+ yield* TestClock.adjust("61 seconds");
329
+ yield* Effect.flatten(Ref.get(runDueWork));
330
+ const [raced] = (yield* service.list()).tasks;
331
+ expect(raced?.lastRunStatus).toBe("failed");
332
+ expect(raced?.lastRunError).toBe("Provider failed");
333
+ expect(Date.parse(raced?.nextRunAt ?? "")).toBeGreaterThan(
334
+ Date.parse(blocked?.nextRunAt ?? ""),
335
+ );
336
+
337
// Run now is an explicit request, so it still queues behind the held one.
338
yield* service.runNow({ id: task.id });
339
expect(yield* queuedOccurrences).toBe(2);
@@ -306,6 +348,9 @@ it.effect("keeps one queued occurrence of a recurring task while its bound threa
348
yield* TestClock.adjust("61 seconds");
349
yield* Effect.flatten(Ref.get(runDueWork));
350
expect(yield* queuedOccurrences).toBe(1);
351
+ const [resumed] = (yield* service.list()).tasks;
352
+ expect(resumed?.lastRunStatus).toBe("succeeded");
353
+ expect(resumed?.lastRunError).toBeNull();
354
355
// A one-time task's only scheduled occurrence still sends behind a queued Run now.
356
const { task: once } = yield* service.upsert(
@@ -314,7 +359,7 @@ it.effect("keeps one queued occurrence of a recurring task while its bound threa
359
title: "Check once",
360
prompt: "Check once",
361
enabled: true,
317
- schedule: { type: "once", runAt: "2026-07-01T09:05:00.000Z" },
362
+ schedule: { type: "once", runAt: "2026-07-01T09:07:00.000Z" },
363
projectId: "project-busy-thread",
364
threadId: "thread:busy",
365
workspaceStrategy: { type: "root" },
apps/server/src/scheduledTasks/ScheduledTaskService.test.ts
+11
-4
@@ -422,9 +422,10 @@ it.effect("thread cleanup notifies subscribers and a finishing run cannot recrea
422
);
423
424
it.effect.each([
425
- { status: "failed", error: "Weekly usage limit reached" },
426
- { status: "interrupted", error: "The run was stopped." },
427
- { status: "cancelled", error: "The run was cancelled." },
425
+ { status: "failed", error: "Weekly usage limit reached", heldWarning: false },
426
+ { status: "failed", error: "Weekly usage limit reached", heldWarning: true },
427
+ { status: "interrupted", error: "The run was stopped.", heldWarning: false },
428
+ { status: "cancelled", error: "The run was cancelled.", heldWarning: false },
429
] as const)("a run that ends $status after its message was accepted records it", (outcome) =>
430
Effect.gen(function* () {
431
const projectId = ProjectId.make("project:test");
@@ -495,9 +496,15 @@ it.effect.each([
496
interactionMode: "default",
497
});
498
yield* service.runNow({ id: task.id });
499
+ if (outcome.heldWarning) {
500
+ const sql = yield* SqlClient.SqlClient;
501
+ yield* sql`UPDATE scheduled_tasks SET last_run_status = 'failed',
502
+ last_run_error = 'A previous occurrence is held in the thread queue. Resume or remove it to run this task again.'
503
+ WHERE task_id = ${task.id}`;
504
+ }
505
const failed = yield* service.subscribeList().pipe(
506
Stream.map(({ tasks }) => tasks.find((candidate) => candidate.id === task.id)),
500
- Stream.filter((current) => current?.lastRunStatus === "failed"),
507
+ Stream.filter((current) => current?.lastRunError === outcome.error),
508
Stream.runHead,
509
Effect.forkChild,
510
);
apps/server/src/scheduledTasks/ScheduledTaskService.ts
+41
-12
@@ -311,6 +311,8 @@ function iso(value: DateTime.DateTime): string {
311
}
312
313
const RUN_OUTCOME_WAIT_MS = 24 * 60 * 60 * 1_000;
314
+const HELD_OCCURRENCE_ERROR =
315
+ "A previous occurrence is held in the thread queue. Resume or remove it to run this task again.";
316
const UNSUCCESSFUL_RUN_ERRORS = {
317
failed: "The run failed.",
318
interrupted: "The run was stopped.",
@@ -848,20 +850,28 @@ export const layer = Layer.effect(
850
851
// Reads every run row but only the queued runs' messages, so the cost per
852
// due interval tracks the thread's run count, not its transcript.
851
- const hasQueuedOccurrence = Effect.fn("ScheduledTaskService.hasQueuedOccurrence")(function* (
853
+ const queuedOccurrence = Effect.fn("ScheduledTaskService.queuedOccurrence")(function* (
854
taskId: ScheduledTaskId,
855
threadId: ThreadId,
856
) {
857
const { runs } = yield* threadManagement.getThreadRecords(threadId, ["runs"]);
858
const queued = runs.filter((run) => run.status === "queued");
857
- if (queued.length === 0) return false;
859
+ if (queued.length === 0) return null;
860
const queuedMessageIds = new Set(queued.map((run) => run.userMessageId));
861
const { messages } = yield* threadManagement.getThreadRecords(threadId, ["messages"], {
862
messageIds: [...queuedMessageIds],
863
});
862
- return messages.some(
863
- (message) => queuedMessageIds.has(message.id) && message.scheduledTaskId === taskId,
864
+ const taskMessageIds = new Set(
865
+ messages
866
+ .filter((message) => message.scheduledTaskId === taskId)
867
+ .map((message) => message.id),
868
);
869
+ const occurrences = queued.filter((run) => taskMessageIds.has(run.userMessageId));
870
+ return occurrences.some((run) => run.queueHeld === true)
871
+ ? "held"
872
+ : occurrences.length > 0
873
+ ? "queued"
874
+ : null;
875
});
876
877
// The run already started the thread, so a failed write only costs the binding.
@@ -940,7 +950,9 @@ export const layer = Layer.effect(
950
const updated = yield* sql`
951
UPDATE scheduled_tasks
952
SET last_run_status = 'failed', last_run_error = ${error}
943
- WHERE task_id = ${taskId} AND last_run_at = ${startedAtIso} AND last_run_status = 'succeeded'
953
+ WHERE task_id = ${taskId} AND last_run_at = ${startedAtIso}
954
+ AND (last_run_status = 'succeeded'
955
+ OR (last_run_status = 'failed' AND last_run_error = ${HELD_OCCURRENCE_ERROR}))
956
RETURNING task_id
957
`;
958
if (updated.length > 0) yield* notifyChanged;
@@ -998,13 +1010,21 @@ export const layer = Layer.effect(
1010
if (
1011
trigger === "scheduled" &&
1012
active.schedule.type !== "once" &&
1001
- active.threadId !== null &&
1002
- (yield* hasQueuedOccurrence(active.id, ThreadId.make(active.threadId)).pipe(
1003
- Effect.orElseSucceed(() => false),
1004
- ))
1013
+ active.threadId !== null
1014
) {
1006
- yield* rescheduleMissedRun(active, startedAt, "previous occurrence still queued");
1007
- return { task: active, threadId: null, dispatchFailed: false };
1015
+ const occurrence = yield* queuedOccurrence(
1016
+ active.id,
1017
+ ThreadId.make(active.threadId),
1018
+ ).pipe(Effect.orElseSucceed(() => null));
1019
+ if (occurrence !== null) {
1020
+ yield* rescheduleMissedRun(
1021
+ active,
1022
+ startedAt,
1023
+ "previous occurrence still queued",
1024
+ occurrence === "held" ? HELD_OCCURRENCE_ERROR : undefined,
1025
+ );
1026
+ return { task: active, threadId: null, dispatchFailed: false };
1027
+ }
1028
}
1029
// Reject queued deliveries after pause, same-id recreation or trigger changes.
1030
if (webhook !== undefined) {
@@ -1254,7 +1274,16 @@ export const layer = Layer.effect(
1274
UPDATE scheduled_tasks
1275
SET next_run_at = ${next},
1276
updated_at = ${iso(now)}
1257
- ${runError === undefined ? sql`` : sql`, last_run_status = 'failed', last_run_error = ${runError}`}
1277
+ ${
1278
+ runError === undefined
1279
+ ? sql``
1280
+ : sql`, last_run_status = CASE
1281
+ WHEN last_run_status IS ${task.lastRunStatus} AND last_run_error IS ${task.lastRunError}
1282
+ THEN 'failed' ELSE last_run_status END,
1283
+ last_run_error = CASE
1284
+ WHEN last_run_status IS ${task.lastRunStatus} AND last_run_error IS ${task.lastRunError}
1285
+ THEN ${runError} ELSE last_run_error END`
1286
+ }
1287
WHERE task_id = ${task.id}
1288
AND next_run_at IS ${task.nextRunAt}
1289
AND last_run_at IS ${task.lastRunAt}
scripts/scenarios/held-scheduled-occurrence.e2e.ts
new
+78
@@ -0,0 +1,78 @@
1
+import { expect } from "e2e";
2
+import * as Effect from "effect/Effect";
3
+import * as FileSystem from "effect/FileSystem";
4
+import * as Path from "effect/Path";
5
+import * as Schema from "effect/Schema";
6
+
7
+import { T3_HOME } from "../e2e.config.ts";
8
+import { runPromise, test, WEB } from "./t3.ts";
9
+
10
+const SettingsJson = Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown));
11
+const decodeSettings = Schema.decodeEffect(SettingsJson);
12
+const encodeSettings = Schema.encodeEffect(SettingsJson);
13
+
14
+const threadRuntimeSettings = Effect.gen(function* () {
15
+ const fs = yield* FileSystem.FileSystem;
16
+ const settingsPath = (yield* Path.Path).join(T3_HOME, "userdata/settings.json");
17
+ const original = yield* fs.readFileString(settingsPath);
18
+ const settings = yield* decodeSettings(original);
19
+ return {
20
+ supervise: fs.writeFileString(
21
+ settingsPath,
22
+ yield* encodeSettings({ ...settings, defaultRuntimeMode: "approval-required" }),
23
+ ),
24
+ restore: fs.writeFileString(settingsPath, original),
25
+ };
26
+});
27
+
28
+test(
29
+ "A recurring task reports when its queued occurrence is held",
30
+ WEB,
31
+ async ({ t3, app, screen }) => {
32
+ const settings = await runPromise(threadRuntimeSettings);
33
+ try {
34
+ // Supervised mode makes the approval recording wait for an answer, which holds its turn open.
35
+ await runPromise(settings.supervise);
36
+ const project = await t3.project("scheduled-hold");
37
+ await t3.thread({ project, title: "Earlier work", messages: ["Start"] });
38
+ const thread = await t3.thread({
39
+ project,
40
+ title: "Waiting scheduled thread",
41
+ messages: ["Ask a question"],
42
+ model: "approval",
43
+ });
44
+
45
+ await t3.open("/settings/scheduled-tasks");
46
+ await screen.getByRole("button", "New task").tap();
47
+ await screen.getByRole("textbox", "Name").fill("Held recurring task");
48
+ await screen.getByRole("combobox", "Destination").tap();
49
+ await screen.getByRole("option", "One thread, continuing the conversation").tap();
50
+ await screen.getByRole("combobox", "Destination thread").tap();
51
+ await screen.getByRole("option", "Waiting scheduled thread").tap();
52
+ await screen.getByRole("textbox", "Prompt").fill("Scheduled check prompt");
53
+ await screen.getByRole("button", "Every interval").tap();
54
+ await screen.getByRole("spinbutton", "Run every").fill("1");
55
+ await screen.getByRole("button", "Create task").tap();
56
+
57
+ await t3.open(thread);
58
+ await screen.getByRole("button", "Approve", { exact: true }).waitFor();
59
+ await screen
60
+ .getByRole("button", "Automation Held recurring task")
61
+ .waitFor({ timeout: 100_000 });
62
+ // Stopping the active turn holds the queued scheduled occurrence.
63
+ await screen.getByRole("button", "Stop generation").tap();
64
+ await screen.getByRole("button", "Resume thread").waitFor();
65
+ await screen.getByRole("button", "Edit queued message").waitFor();
66
+
67
+ await app.open("/settings/scheduled-tasks");
68
+ await expect(
69
+ screen.getByText(
70
+ "A previous occurrence is held in the thread queue. Resume or remove it to run this task again.",
71
+ ),
72
+ ).toBeVisible({ timeout: 100_000 });
73
+ await app.screenshot("held-scheduled-task");
74
+ } finally {
75
+ await runPromise(settings.restore);
76
+ }
77
+ },
78
+);