@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 2,694 lines 111 KB
Raw
1 import {
2 CommandId,
3 MessageId,
4 PROVIDER_SEND_TURN_MAX_INPUT_CHARS,
5 type ProjectId,
6 ScheduledTask,
7 ScheduledTaskError,
8 ScheduledTaskId,
9 ThreadId,
10 ScheduledTaskWebhookDeliveryId,
11 type ScheduledTaskDeleteInput,
12 type ScheduledTaskDeleteResult,
13 type ScheduledTaskGetWebhookDeliveryInput,
14 type ScheduledTaskGetWebhookDeliveryResult,
15 type ScheduledTaskListWebhookDeliveriesInput,
16 type ScheduledTaskListWebhookDeliveriesResult,
17 type ScheduledTaskRotateWebhookTokenInput,
18 type ScheduledTaskWebhookDeliveryOutcome,
19 type ScheduledTaskWebhookFilter,
20 type ScheduledTaskListResult,
21 type ScheduledTaskMutationResult,
22 type ScheduledTaskRunNowInput,
23 type ScheduledTaskRunNowResult,
24 type ScheduledTaskSetEnabledInput,
25 type ScheduledTaskThreadMode,
26 type ScheduledTaskUpsertInput,
27 managedPolicyDeniedMessage,
28 scheduledTaskThreadMode,
29 managedRuntimeModeDenial,
30 } from "@t3tools/contracts";
31 import * as Cause from "effect/Cause";
32 import * as Context from "effect/Context";
33 import * as Digest from "effect/crypto/Digest";
34 import * as Mac from "effect/crypto/Mac";
35 import * as SecureRandom from "effect/crypto/SecureRandom";
36 import * as DateTime from "effect/DateTime";
37 import * as Effect from "effect/Effect";
38 import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient";
39 import * as Exit from "effect/Exit";
40 import * as Duration from "effect/Duration";
41 import * as Metric from "effect/Metric";
42 import * as Layer from "effect/Layer";
43 import * as Option from "effect/Option";
44 import * as PubSub from "effect/PubSub";
45 import * as Ref from "effect/Ref";
46 import * as Result from "effect/Result";
47 import * as Schema from "effect/Schema";
48 import * as Semaphore from "effect/Semaphore";
49 import * as Stream from "effect/Stream";
50 import * as SqlClient from "effect/sql/SqlClient";
51
52 import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts";
53 import * as ManagedProjectFolders from "../project/ManagedProjectFolders.ts";
54 import * as Metrics from "../observability/Metrics.ts";
55 import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts";
56 import { OrchestratorDispatchError } from "../orchestration-v2/Orchestrator.ts";
57 import { userFacingDispatchErrorMessage } from "../orchestration-v2/UserFacingErrors.ts";
58 import * as SecretRequests from "../secrets/SecretRequests.ts";
59 import * as ManagedPolicy from "../policy/ManagedPolicy.ts";
60 import * as DueHint from "../scheduling/DueHint.ts";
61 import * as Scheduler from "../scheduling/Scheduler.ts";
62 import { latestRootProviderFailure } from "@t3tools/shared/orchestrationV2ThreadError";
63 import { isMissedFixedTimeRun, isSameSchedule, nextScheduledRunAt } from "./Schedule.ts";
64 import {
65 redactHeaders,
66 redactQuery,
67 renderWebhookPrompt,
68 type WebhookRequest,
69 } from "./webhookTemplate.ts";
70 import {
71 acceptWebhookDedupe,
72 acceptImmediateWebhookDedupe,
73 releaseWebhookReplay,
74 decodePendingTask,
75 encodePendingTask,
76 decodePendingRequest,
77 dedupeError,
78 } from "./webhookDedupe.ts";
79 import { describeFailedWebhookFilter, findFailedWebhookFilter } from "./webhookFilter.ts";
80 import { constantTimeEquals, verifyWebhookSignature } from "./webhookVerification.ts";
81
82 export const WEBHOOK_ROUTE_PREFIX = "/api/hooks";
83 const isScheduledTaskError = Schema.is(ScheduledTaskError);
84 const isOrchestratorDispatchError = Schema.is(OrchestratorDispatchError);
85 const PROMPT_TOO_LONG = "The filled-in prompt is too long.";
86 const WEBHOOK_DELIVERY_RETENTION = 50;
87 const WEBHOOK_DEDUPE_DELIVERY_RETENTION = 20;
88 const WEBHOOK_FILTERED_DELIVERY_RETENTION = 20;
89 const WEBHOOK_DEDUPE_LOG_PER_MINUTE = 10;
90 const WEBHOOK_FILTERED_LOG_PER_MINUTE = 10;
91 const WEBHOOK_DELIVERY_LOG_BODY_LIMIT = 64 * 1024;
92 const WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT = 64 * 1024;
93 const WEBHOOK_MAX_QUEUED_PER_TASK = 20;
94 // Enforce the relay's per-task/minute limit here too; tunnel hostnames are public.
95 const WEBHOOK_RATE_LIMIT_PER_MINUTE = 60;
96 const LEGACY_DELIVERY_OUTCOMES: ReadonlySet<string> = new Set([
97 "accepted",
98 "dispatch_failed",
99 "rejected_signature",
100 "disabled",
101 "rate_limited",
102 "expired",
103 ]);
104
105 interface WebhookOrigin {
106 readonly relayHookBaseUrl: string | null;
107 }
108
109 const ENDPOINT_KEY = /^[0-9a-f]{16}$/;
110
111 export function relayHookBaseUrl(input: {
112 readonly relayUrl: string;
113 readonly tunnelName: string | undefined;
114 }): string | null {
115 const endpointKey = input.tunnelName?.split("-").at(-1);
116 const relayUrl = input.relayUrl.replace(/\/+$/, "");
117 if (relayUrl === "" || endpointKey === undefined || !ENDPOINT_KEY.test(endpointKey)) {
118 return null;
119 }
120 return `${relayUrl}/v1/hooks/${endpointKey}`;
121 }
122
123 export class ScheduledTaskWebhookOrigin extends Context.Reference<Effect.Effect<WebhookOrigin>>(
124 "t3/scheduledTasks/ScheduledTaskWebhookOrigin",
125 {
126 defaultValue: () => Effect.succeed({ relayHookBaseUrl: null }),
127 },
128 ) {}
129
130 class WebhookDeliverySkipped extends Schema.TaggedError<WebhookDeliverySkipped>()(
131 "WebhookDeliverySkipped",
132 { reason: Schema.String },
133 ) {}
134
135 interface RateWindow {
136 readonly accepted: ReadonlyArray<{ readonly at: number }>;
137 readonly rejectedLogged: boolean;
138 }
139
140 export interface WebhookTriggerRequest extends WebhookRequest {
141 readonly hookId: string;
142 readonly token: string;
143 readonly body: Uint8Array;
144 readonly relayDeliveryId?: string;
145 readonly receivedAt?: string;
146 }
147
148 export type WebhookDeliveryOutcome =
149 | "accepted"
150 | "duplicate"
151 | "coalesced"
152 | "prompt_too_long"
153 | "queue_full"
154 | "rate_limited"
155 | "disabled"
156 | "rejected_signature"
157 | "expired"
158 | "filtered"
159 | "not_found"
160 | "blocked_by_policy"
161 | "error";
162
163 export type WebhookTriggerResult =
164 | {
165 readonly _tag: "accepted";
166 readonly deliveryId: ScheduledTaskWebhookDeliveryId;
167 readonly outcome: "accepted" | "duplicate" | "coalesced" | "prompt_too_long" | "filtered";
168 }
169 | { readonly _tag: "not_found" }
170 | { readonly _tag: "rejected_signature" }
171 | { readonly _tag: "disabled" }
172 | { readonly _tag: "blocked_by_policy" }
173 | {
174 readonly _tag: "rate_limited";
175 readonly outcome: "rate_limited" | "queue_full";
176 }
177 | { readonly _tag: "expired" };
178
179 const webhookFilters = (filters: ReadonlyArray<ScheduledTaskWebhookFilter> | undefined) =>
180 filters?.length ? { filters } : {};
181
182 const decodeTask = Schema.decodeUnknownEffect(ScheduledTask);
183 const decodeTaskId = Schema.decodeUnknownOption(ScheduledTaskId);
184 const decodeScheduleJson = Schema.decodeUnknownEffect(
185 Schema.fromJsonString(ScheduledTask.fields.schedule),
186 );
187 const decodeWorkspaceStrategyJson = Schema.decodeUnknownEffect(
188 Schema.fromJsonString(ScheduledTask.fields.workspaceStrategy),
189 );
190 const decodeModelSelectionJson = Schema.decodeUnknownEffect(
191 Schema.fromJsonString(ScheduledTask.fields.modelSelection),
192 );
193 const HeadersJson = Schema.fromJsonString(Schema.Record(Schema.String, Schema.String));
194 const MissingFieldsJson = Schema.fromJsonString(Schema.Array(Schema.String));
195 const decodeHeadersJson = Schema.decodeUnknownOption(HeadersJson);
196 const decodeMissingFieldsJson = Schema.decodeUnknownOption(MissingFieldsJson);
197 const encodeHeadersJson = Schema.encodeSync(HeadersJson);
198 const encodeMissingFieldsJson = Schema.encodeSync(MissingFieldsJson);
199
200 /**
201 * Clients that predate thread modes send none, and may send a thread loaded before the first run
202 * bound one. An edit keeps the stored mode; binding a thread to a new-thread task continues it.
203 */
204 function resolveThreadMode(
205 input: Pick<ScheduledTaskUpsertInput, "threadId" | "threadMode">,
206 existing: Pick<ScheduledTask, "threadId" | "threadMode"> | null,
207 ): ScheduledTaskThreadMode {
208 if (input.threadMode !== undefined) return input.threadMode;
209 const stored = existing === null ? "new_thread" : scheduledTaskThreadMode(existing);
210 return stored === "new_thread" && input.threadId != null ? "continue" : stored;
211 }
212
213 interface ScheduledTaskRow {
214 readonly task_id: string;
215 readonly title: string;
216 readonly prompt: string;
217 readonly enabled: number;
218 readonly schedule_json: string;
219 readonly project_id: string;
220 readonly thread_id: string | null;
221 readonly thread_mode: string;
222 readonly workspace_strategy_json: string;
223 readonly model_selection_json: string;
224 readonly runtime_mode: string;
225 readonly interaction_mode: string;
226 readonly created_by: string;
227 readonly creation_source: string;
228 readonly created_at: string;
229 readonly updated_at: string;
230 readonly next_run_at: string | null;
231 readonly last_run_at: string | null;
232 readonly last_run_status: string;
233 readonly last_run_error: string | null;
234 readonly run_count: number;
235 readonly webhook_token: string | null;
236 readonly webhook_secret: string | null;
237 }
238
239 interface WebhookDeliveryRow {
240 readonly delivery_id: string;
241 readonly task_id: string;
242 readonly received_at: string;
243 readonly method: string;
244 readonly query: string;
245 readonly headers_json: string;
246 readonly body: string;
247 readonly body_bytes: number;
248 readonly body_truncated: number;
249 readonly outcome: string;
250 readonly signature_verified: number;
251 readonly missing_fields_json: string;
252 readonly rendered_prompt: string | null;
253 readonly pending_window: number;
254 readonly error: string | null;
255 }
256
257 export class ScheduledTaskService extends Context.Service<
258 ScheduledTaskService,
259 {
260 readonly list: () => Effect.Effect<ScheduledTaskListResult, ScheduledTaskError>;
261 readonly subscribeList: () => Stream.Stream<ScheduledTaskListResult, ScheduledTaskError>;
262 readonly upsert: (
263 input: ScheduledTaskUpsertInput,
264 ) => Effect.Effect<ScheduledTaskMutationResult, ScheduledTaskError>;
265 readonly setEnabled: (
266 input: ScheduledTaskSetEnabledInput,
267 ) => Effect.Effect<ScheduledTaskMutationResult, ScheduledTaskError>;
268 readonly delete: (
269 input: ScheduledTaskDeleteInput,
270 ) => Effect.Effect<ScheduledTaskDeleteResult, ScheduledTaskError>;
271 /** Deletes only tasks that still target this thread, including their webhook deliveries. */
272 readonly deleteForThread: (threadId: ThreadId) => Effect.Effect<void, ScheduledTaskError>;
273 /** Points tasks bound to these threads at the project the threads moved to. */
274 readonly moveThreadTasks: (input: {
275 readonly threadIds: ReadonlyArray<ThreadId>;
276 readonly projectId: ProjectId;
277 }) => Effect.Effect<void, ScheduledTaskError>;
278 /** The dispatched thread, or null if dispatch failed; leaves the task's binding unchanged. */
279 readonly runNow: (
280 input: ScheduledTaskRunNowInput,
281 ) => Effect.Effect<
282 ScheduledTaskRunNowResult & { readonly threadId: ThreadId | null },
283 ScheduledTaskError
284 >;
285 readonly rotateWebhookToken: (
286 input: ScheduledTaskRotateWebhookTokenInput,
287 ) => Effect.Effect<ScheduledTaskMutationResult, ScheduledTaskError>;
288 readonly listWebhookDeliveries: (
289 input: ScheduledTaskListWebhookDeliveriesInput,
290 ) => Effect.Effect<ScheduledTaskListWebhookDeliveriesResult, ScheduledTaskError>;
291 readonly getWebhookDelivery: (
292 input: ScheduledTaskGetWebhookDeliveryInput,
293 ) => Effect.Effect<ScheduledTaskGetWebhookDeliveryResult, ScheduledTaskError>;
294 /** Verifies, logs and dispatches; returns after logging while the run continues in the background. */
295 readonly triggerWebhook: (
296 request: WebhookTriggerRequest,
297 ) => Effect.Effect<WebhookTriggerResult, ScheduledTaskError>;
298 }
299 >()("t3/scheduledTasks/ScheduledTaskService") {}
300
301 function taskError(message: string, input?: { taskId?: ScheduledTaskId; cause?: unknown }) {
302 return new ScheduledTaskError({
303 message,
304 ...(input?.taskId === undefined ? {} : { taskId: input.taskId }),
305 ...(input?.cause === undefined ? {} : { cause: input.cause }),
306 });
307 }
308
309 function iso(value: DateTime.DateTime): string {
310 return DateTime.formatIso(DateTime.toUtc(value));
311 }
312
313 const RUN_OUTCOME_WAIT_MS = 24 * 60 * 60 * 1_000;
314 const HELD_OCCURRENCE_ERROR =
315 "A previous occurrence is held in the thread queue. Resume or remove it to run this task again.";
316 const UNSUCCESSFUL_RUN_ERRORS = {
317 failed: "The run failed.",
318 interrupted: "The run was stopped.",
319 cancelled: "The run was cancelled.",
320 rolled_back: "The run was rolled back.",
321 } as const;
322 const localNow = DateTime.withCurrentZoneLocal(DateTime.nowInCurrentZone);
323
324 function nextRunAt(
325 task: Pick<ScheduledTask, "enabled" | "schedule">,
326 from: DateTime.DateTime,
327 ): string | null {
328 if (!task.enabled) return null;
329 // Decoded intervals can overflow DateTime; unrepresentable occurrences have no next run.
330 try {
331 const next = nextScheduledRunAt(task.schedule, from);
332 return next !== null && Number.isFinite(DateTime.toEpochMillis(next)) ? iso(next) : null;
333 } catch {
334 return null;
335 }
336 }
337
338 /**
339 * Next run after an interrupted run. A one-time task counts from the run's start
340 * (`lastRunAt`), so a manual run that began before its time does not use it up.
341 */
342 function nextRunAfterRun(
343 task: Pick<ScheduledTask, "enabled" | "schedule" | "lastRunAt">,
344 now: DateTime.DateTime,
345 ): string | null {
346 const startedAt =
347 task.schedule.type === "once" && task.lastRunAt !== null
348 ? DateTime.make(task.lastRunAt)
349 : Option.none();
350 return nextRunAt(
351 task,
352 Option.getOrElse(startedAt, () => now),
353 );
354 }
355
356 /** An enabled one-time task with a past time would never run, so saving it is an error. */
357 function requireFutureOnce(
358 task: Pick<ScheduledTask, "enabled" | "schedule">,
359 now: DateTime.DateTime,
360 taskId: ScheduledTaskId,
361 ) {
362 return task.enabled && task.schedule.type === "once" && nextRunAt(task, now) === null
363 ? taskError("The run time has passed. Pick a time in the future.", { taskId })
364 : Effect.void;
365 }
366
367 function errorMessage(error: unknown): string {
368 const failure = Cause.isCause(error) ? Cause.squash(error) : error;
369 return failure instanceof Error ? failure.message : String(failure);
370 }
371
372 function dispatchErrorMessage(cause: Cause.Cause<unknown>): string {
373 const failure = Cause.squash(cause);
374 // Direct orchestration refusals are user-facing; nested defects can contain private data.
375 return isOrchestratorDispatchError(failure) && typeof failure.cause === "string"
376 ? (userFacingDispatchErrorMessage(failure.cause) ?? errorMessage(failure))
377 : errorMessage(failure);
378 }
379
380 function webhookPath(taskId: string, token: string): string {
381 return `${WEBHOOK_ROUTE_PREFIX}/${encodeURIComponent(taskId)}/${token}`;
382 }
383
384 function webhookEndpoint(
385 row: Pick<ScheduledTaskRow, "task_id" | "webhook_token" | "webhook_secret">,
386 origin: WebhookOrigin | null,
387 ): ScheduledTask["webhook"] {
388 if (row.webhook_token === null) return undefined;
389 const base = origin?.relayHookBaseUrl ?? null;
390 return {
391 path: webhookPath(row.task_id, row.webhook_token),
392 url: base === null ? null : `${base}/${encodeURIComponent(row.task_id)}/${row.webhook_token}`,
393 hasSecret: row.webhook_secret !== null,
394 };
395 }
396
397 // Runs per task row; list and lookup operations already own the span.
398 const decodeRow = Effect.fnUntraced(
399 function* (row: ScheduledTaskRow, origin: WebhookOrigin | null = null) {
400 const schedule = yield* decodeScheduleJson(row.schedule_json);
401 const webhook = schedule.type === "webhook" ? webhookEndpoint(row, origin) : undefined;
402 const workspaceStrategy = yield* decodeWorkspaceStrategyJson(row.workspace_strategy_json);
403 const modelSelection = yield* decodeModelSelectionJson(row.model_selection_json);
404 return yield* decodeTask({
405 id: row.task_id,
406 title: row.title,
407 prompt: row.prompt,
408 enabled: row.enabled === 1,
409 schedule,
410 projectId: row.project_id,
411 threadId: row.thread_id,
412 threadMode: row.thread_mode,
413 workspaceStrategy,
414 modelSelection,
415 runtimeMode: row.runtime_mode,
416 interactionMode: row.interaction_mode,
417 createdBy: row.created_by,
418 creationSource: row.creation_source,
419 createdAt: row.created_at,
420 updatedAt: row.updated_at,
421 nextRunAt: row.next_run_at,
422 lastRunAt: row.last_run_at,
423 lastRunStatus: row.last_run_status,
424 lastRunError: row.last_run_error,
425 runCount: row.run_count,
426 ...(webhook === undefined ? {} : { webhook }),
427 });
428 },
429 (effect, row, _origin?) =>
430 effect.pipe(
431 Effect.mapError((cause) => {
432 const taskId = decodeTaskId(row.task_id);
433 return taskError("Could not decode schedule task row.", {
434 ...(Option.isSome(taskId) ? { taskId: taskId.value } : {}),
435 cause,
436 });
437 }),
438 ),
439 );
440
441 export const listDueTasks = Effect.fn("ScheduledTaskService.listDueTasks")(function* (
442 now: DateTime.DateTime,
443 ) {
444 const sql = yield* SqlClient.SqlClient;
445 const rows = yield* sql<ScheduledTaskRow>`
446 SELECT * FROM scheduled_tasks
447 WHERE enabled = 1 AND next_run_at IS NOT NULL
448 AND next_run_at <= ${iso(now)} AND last_run_status <> 'running'
449 ORDER BY next_run_at ASC, task_id ASC
450 `.pipe(NodeSqliteClient.readOnly);
451 const tasks: ScheduledTask[] = [];
452 for (const row of rows) {
453 const decoded = yield* Effect.result(decodeRow(row));
454 if (Result.isSuccess(decoded)) {
455 const task = decoded.success;
456 // next_run_at is freeform; skip unparseable values before they defect the poll.
457 if (task.nextRunAt === null || Option.isSome(DateTime.make(task.nextRunAt))) {
458 tasks.push(task);
459 } else {
460 yield* Effect.logWarning("Skipping schedule task row with invalid next_run_at", {
461 taskId: row.task_id,
462 });
463 }
464 } else {
465 yield* Effect.logWarning("Skipping undecodable schedule task row", {
466 taskId: row.task_id,
467 cause: decoded.failure,
468 });
469 }
470 }
471 return tasks;
472 });
473
474 export const layer = Layer.effect(
475 ScheduledTaskService,
476 Effect.gen(function* () {
477 const sql = yield* SqlClient.SqlClient;
478 const random = yield* SecureRandom.SecureRandom;
479 const digest = yield* Digest.Digest;
480 const mac = yield* Mac.Mac;
481 const threadLaunch = yield* ThreadLaunchService.ThreadLaunchService;
482 const threadManagement = yield* ThreadManagementService.ThreadManagementService;
483 const serviceScope = yield* Effect.scope;
484 const managedFolders = yield* ManagedProjectFolders.ManagedProjectFolders;
485 const secretRequests = yield* SecretRequests.SecretRequests;
486 const scheduler = yield* Scheduler.Scheduler;
487 const readWebhookOrigin = yield* ScheduledTaskWebhookOrigin;
488 const managedPolicy = yield* ManagedPolicy.ManagedPolicy;
489 const webhookPolicyOrganization = ManagedPolicy.deniedBy(managedPolicy, "webhookTasks");
490 // Existing webhook tasks stay stored but report why they do not run.
491 const webhookPolicyBlock =
492 webhookPolicyOrganization === undefined
493 ? undefined
494 : managedPolicyDeniedMessage(webhookPolicyOrganization, "webhook tasks");
495 const withPolicy = (task: ScheduledTask): ScheduledTask =>
496 webhookPolicyBlock !== undefined && task.schedule.type === "webhook"
497 ? { ...task, blockedByPolicy: webhookPolicyBlock }
498 : task;
499 const webhookPermits = yield* Ref.make<ReadonlyMap<ScheduledTaskId, Semaphore.Semaphore>>(
500 new Map(),
501 );
502 // Include creation time so late deliveries cannot decrement a recreated task's count.
503 const webhookQueued = yield* Ref.make<ReadonlyMap<string, number>>(new Map());
504 const webhookRateWindows = yield* Ref.make<ReadonlyMap<ScheduledTaskId, RateWindow>>(new Map());
505 const webhookRejectedRateWindows = yield* Ref.make<ReadonlyMap<ScheduledTaskId, RateWindow>>(
506 new Map(),
507 );
508 const webhookDedupeLogWindows = yield* Ref.make<
509 ReadonlyMap<ScheduledTaskId, ReadonlyArray<number>>
510 >(new Map());
511 const pendingInFlight = yield* Ref.make<ReadonlySet<string>>(new Set());
512 const webhookFilteredLogWindows = yield* Ref.make<
513 ReadonlyMap<ScheduledTaskId, ReadonlyArray<number>>
514 >(new Map());
515 const activeRuns = yield* Ref.make<ReadonlySet<ScheduledTaskId>>(new Set());
516 // Notifications re-emit the full list; slow subscribers need only the newest dirty signal.
517 const changesPubSub = yield* PubSub.sliding<void>(1);
518 const dueTasks = DueHint.make();
519 const pendingWebhooks = DueHint.make();
520 // Every scheduled_tasks write that can change next_run_at, enabled or run state notifies.
521 const notifyChanged = dueTasks.invalidate.pipe(
522 Effect.andThen(PubSub.publish(changesPubSub, undefined)),
523 Effect.asVoid,
524 );
525
526 const selectAllRows = () =>
527 sql<ScheduledTaskRow>`
528 SELECT
529 task_id,
530 title,
531 prompt,
532 enabled,
533 schedule_json,
534 project_id,
535 thread_id,
536 thread_mode,
537 workspace_strategy_json,
538 model_selection_json,
539 runtime_mode,
540 interaction_mode,
541 created_by,
542 creation_source,
543 created_at,
544 updated_at,
545 next_run_at,
546 last_run_at,
547 last_run_status,
548 last_run_error,
549 run_count,
550 webhook_token,
551 webhook_secret
552 FROM scheduled_tasks
553 -- Not updated_at: toggles, runs and edits rewrite it, and clients render
554 -- this order as-is, so rows would move under the pointer.
555 ORDER BY created_at DESC, task_id ASC
556 `.pipe(NodeSqliteClient.readOnly);
557
558 const listRows = Effect.fn("ScheduledTaskService.listRows")(function* () {
559 const rows = yield* selectAllRows();
560 const origin = yield* readWebhookOrigin;
561 return yield* Effect.forEach(
562 rows,
563 (row) => decodeRow(row, origin).pipe(Effect.map(withPolicy)),
564 { concurrency: 1 },
565 );
566 });
567
568 const getRows = (id: ScheduledTaskId) =>
569 sql<ScheduledTaskRow>`
570 SELECT
571 task_id,
572 title,
573 prompt,
574 enabled,
575 schedule_json,
576 project_id,
577 thread_id,
578 thread_mode,
579 workspace_strategy_json,
580 model_selection_json,
581 runtime_mode,
582 interaction_mode,
583 created_by,
584 creation_source,
585 created_at,
586 updated_at,
587 next_run_at,
588 last_run_at,
589 last_run_status,
590 last_run_error,
591 run_count,
592 webhook_token,
593 webhook_secret
594 FROM scheduled_tasks
595 WHERE task_id = ${id}
596 `.pipe(NodeSqliteClient.readOnly);
597
598 const findTask = Effect.fn("ScheduledTaskService.findTask")(function* (id: ScheduledTaskId) {
599 const rows = yield* getRows(id).pipe(
600 Effect.mapError((cause) =>
601 taskError("Could not load schedule task.", { taskId: id, cause }),
602 ),
603 );
604 const row = rows[0];
605 if (row === undefined) return null;
606 return withPolicy(yield* decodeRow(row, yield* readWebhookOrigin));
607 });
608
609 const findWebhookCredentials = (id: ScheduledTaskId) =>
610 getRows(id).pipe(
611 Effect.map((rows) =>
612 rows[0] === undefined
613 ? null
614 : { token: rows[0].webhook_token, secret: rows[0].webhook_secret },
615 ),
616 Effect.mapError((cause) =>
617 taskError("Could not load schedule task.", { taskId: id, cause }),
618 ),
619 );
620
621 const newWebhookToken = random.randomBytes(32).pipe(
622 Effect.map((bytes) => Buffer.from(bytes).toString("base64url")),
623 Effect.mapError((cause) => taskError("Could not generate webhook token.", { cause })),
624 );
625
626 const loadTask = Effect.fn("ScheduledTaskService.loadTask")(function* (id: ScheduledTaskId) {
627 const task = yield* findTask(id);
628 if (task === null) {
629 return yield* taskError("Schedule task not found.", { taskId: id });
630 }
631 return task;
632 });
633
634 // Run transitions own run-state columns; settings saves must not overwrite them.
635 // Check existence in the write so edits cannot undo concurrent deletion.
636 const saveTask = (
637 task: ScheduledTask,
638 requireExisting: boolean,
639 /** The schedule this save read, when it left the schedule unchanged. */
640 keptSchedule: ScheduledTask["schedule"] | null,
641 /** The thread this save unbinds; any other stored thread is kept. */
642 unboundThreadId: ThreadId | null,
643 webhook: {
644 readonly token: string | null;
645 readonly secret: string | null;
646 readonly secretChanged: boolean;
647 },
648 ) =>
649 sql<{ task_id: string }>`
650 INSERT INTO scheduled_tasks (
651 task_id,
652 title,
653 prompt,
654 enabled,
655 schedule_json,
656 project_id,
657 thread_id,
658 thread_mode,
659 workspace_strategy_json,
660 model_selection_json,
661 runtime_mode,
662 interaction_mode,
663 created_by,
664 creation_source,
665 created_at,
666 updated_at,
667 next_run_at,
668 last_run_at,
669 last_run_status,
670 last_run_error,
671 run_count,
672 webhook_token,
673 webhook_secret
674 )
675 SELECT
676 ${task.id},
677 ${task.title},
678 ${task.prompt},
679 ${task.enabled ? 1 : 0},
680 ${JSON.stringify(task.schedule)},
681 ${task.projectId},
682 ${task.threadId},
683 ${scheduledTaskThreadMode(task)},
684 ${JSON.stringify(task.workspaceStrategy)},
685 ${JSON.stringify(task.modelSelection)},
686 ${task.runtimeMode},
687 ${task.interactionMode},
688 ${task.createdBy},
689 ${task.creationSource},
690 ${task.createdAt},
691 ${task.updatedAt},
692 ${task.nextRunAt},
693 ${task.lastRunAt},
694 ${task.lastRunStatus},
695 ${task.lastRunError},
696 ${task.runCount},
697 ${webhook.token},
698 ${webhook.secret}
699 WHERE ${requireExisting ? 0 : 1} = 1
700 OR EXISTS (SELECT 1 FROM scheduled_tasks WHERE task_id = ${task.id})
701 ON CONFLICT (task_id)
702 DO UPDATE SET
703 title = excluded.title,
704 prompt = excluded.prompt,
705 enabled = excluded.enabled,
706 schedule_json = excluded.schedule_json,
707 project_id = excluded.project_id,
708 -- A first run may bind the thread while this save is in flight, so a save
709 -- without a thread keeps any thread but the one it unbinds.
710 thread_id = CASE
711 WHEN excluded.thread_mode = 'new_thread' THEN NULL
712 WHEN excluded.project_id != scheduled_tasks.project_id THEN excluded.thread_id
713 ELSE COALESCE(excluded.thread_id, NULLIF(scheduled_tasks.thread_id, ${unboundThreadId}))
714 END,
715 thread_mode = excluded.thread_mode,
716 workspace_strategy_json = excluded.workspace_strategy_json,
717 model_selection_json = excluded.model_selection_json,
718 runtime_mode = excluded.runtime_mode,
719 interaction_mode = excluded.interaction_mode,
720 creation_source = excluded.creation_source,
721 updated_at = excluded.updated_at,
722 -- Keep a next_run_at that a finishing run moved during this save, unless
723 -- another edit changed the schedule or enabled state meanwhile.
724 next_run_at = CASE
725 WHEN ${keptSchedule === null ? 0 : 1} = 1
726 AND scheduled_tasks.schedule_json IN (
727 excluded.schedule_json,
728 ${keptSchedule === null ? null : JSON.stringify(keptSchedule)}
729 )
730 AND scheduled_tasks.enabled = excluded.enabled
731 THEN scheduled_tasks.next_run_at
732 ELSE excluded.next_run_at
733 END,
734 -- Only rotate changes a live token, so a save racing a rotation
735 -- cannot bring the old URL back.
736 webhook_token = CASE
737 WHEN excluded.webhook_token IS NULL THEN NULL
738 ELSE COALESCE(scheduled_tasks.webhook_token, excluded.webhook_token)
739 END,
740 webhook_secret = CASE
741 WHEN ${webhook.secretChanged ? 1 : 0} = 1 THEN excluded.webhook_secret
742 ELSE scheduled_tasks.webhook_secret
743 END
744 RETURNING task_id
745 `.pipe(
746 Effect.mapError((cause) =>
747 taskError("Could not save schedule task.", { taskId: task.id, cause }),
748 ),
749 Effect.flatMap((rows) =>
750 rows.length > 0
751 ? Effect.void
752 : taskError("Schedule task not found.", { taskId: task.id }),
753 ),
754 );
755
756 const deleteRow = (id: ScheduledTaskId) =>
757 sql
758 .withTransaction(
759 sql`DELETE FROM scheduled_task_webhook_deliveries WHERE task_id = ${id}`.pipe(
760 Effect.andThen(sql`DELETE FROM scheduled_tasks WHERE task_id = ${id}`),
761 ),
762 )
763 .pipe(
764 Effect.mapError((cause) =>
765 taskError("Could not delete schedule task.", { taskId: id, cause }),
766 ),
767 );
768
769 const markRunning = (id: ScheduledTaskId, startedAtIso: string, webhookDeliveryId?: string) =>
770 sql<{ task_id: string }>`
771 UPDATE scheduled_tasks
772 SET updated_at = ${startedAtIso},
773 last_run_at = ${startedAtIso},
774 last_run_webhook_delivery_id = ${webhookDeliveryId ?? null},
775 last_run_status = 'running',
776 last_run_error = NULL
777 WHERE task_id = ${id}
778 RETURNING task_id
779 `.pipe(
780 Effect.mapError((cause) =>
781 taskError("Could not mark schedule task as running.", { taskId: id, cause }),
782 ),
783 Effect.flatMap((rows) =>
784 rows.length > 0 ? Effect.void : taskError("Schedule task not found.", { taskId: id }),
785 ),
786 );
787
788 const markCompleted = (input: {
789 readonly id: ScheduledTaskId;
790 readonly completedAtIso: string;
791 readonly nextRunAtIso: string | null;
792 readonly status: "succeeded" | "failed";
793 readonly error: string | null;
794 readonly startedAtIso: string;
795 }) =>
796 sql`
797 UPDATE scheduled_tasks
798 SET updated_at = ${input.completedAtIso},
799 next_run_at = ${input.nextRunAtIso},
800 last_run_status = ${input.status},
801 last_run_error = ${input.error},
802 run_count = run_count + 1
803 WHERE task_id = ${input.id}
804 AND last_run_status = 'running'
805 AND last_run_at = ${input.startedAtIso}
806 RETURNING task_id
807 `.pipe(
808 Effect.map((rows) => rows.length > 0),
809 Effect.mapError((cause) =>
810 taskError("Could not record schedule task run.", { taskId: input.id, cause }),
811 ),
812 );
813
814 // A failure between markRunning and markCompleted must still write a terminal record:
815 // runDueTasks skips 'running' rows forever, and the dispatch may already have gone out,
816 // so next_run_at must advance. Pending webhook attempts count only with their completion.
817 const releaseStuckRun = Effect.fn("ScheduledTaskService.releaseStuckRun")(
818 function* (
819 task: ScheduledTask,
820 startedAtIso: string,
821 message: string,
822 pendingDeliveryId?: string,
823 ) {
824 const now = yield* localNow;
825 const reread = yield* Effect.result(findTask(task.id));
826 if (Result.isSuccess(reread) && reread.success === null) return;
827 const source = Result.isSuccess(reread) && reread.success !== null ? reread.success : task;
828 yield* sql`
829 UPDATE scheduled_tasks
830 SET last_run_status = 'failed',
831 last_run_error = ${message},
832 next_run_at = ${nextRunAfterRun({ ...source, lastRunAt: startedAtIso }, now)},
833 updated_at = ${iso(now)},
834 run_count = run_count + CASE WHEN EXISTS (
835 SELECT 1 FROM scheduled_task_webhook_pending
836 WHERE delivery_id = ${pendingDeliveryId ?? null} AND state = 'running'
837 ) THEN 0 ELSE 1 END
838 WHERE task_id = ${task.id} AND last_run_status = 'running'
839 `;
840 yield* notifyChanged;
841 },
842 (effect, task, _startedAtIso, _message, _pendingDeliveryId) =>
843 effect.pipe(
844 Effect.catch((cause) =>
845 Effect.logWarning("Could not release stuck schedule task run", {
846 taskId: task.id,
847 cause,
848 }),
849 ),
850 ),
851 );
852
853 // Reads every run row but only the queued runs' messages, so the cost per
854 // due interval tracks the thread's run count, not its transcript.
855 const queuedOccurrence = Effect.fn("ScheduledTaskService.queuedOccurrence")(function* (
856 taskId: ScheduledTaskId,
857 threadId: ThreadId,
858 ) {
859 const { runs } = yield* threadManagement.getThreadRecords(threadId, ["runs"]);
860 const queued = runs.filter((run) => run.status === "queued");
861 if (queued.length === 0) return null;
862 const queuedMessageIds = new Set(queued.map((run) => run.userMessageId));
863 const { messages } = yield* threadManagement.getThreadRecords(threadId, ["messages"], {
864 messageIds: [...queuedMessageIds],
865 });
866 const taskMessageIds = new Set(
867 messages
868 .filter((message) => message.scheduledTaskId === taskId)
869 .map((message) => message.id),
870 );
871 const occurrences = queued.filter((run) => taskMessageIds.has(run.userMessageId));
872 return occurrences.some((run) => run.queueHeld === true)
873 ? "held"
874 : occurrences.length > 0
875 ? "queued"
876 : null;
877 });
878
879 // The run already started the thread, so a failed write only costs the binding.
880 // A thread deleted before the binding landed already ran its task cleanup, so apply that
881 // cleanup here; a later deletion finds the binding. An unconfirmed lookup keeps the binding.
882 const bindThread = Effect.fn("ScheduledTaskService.bindThread")(
883 function* (task: ScheduledTask, threadId: ThreadId) {
884 yield* sql`
885 UPDATE scheduled_tasks SET thread_id = ${threadId}
886 WHERE task_id = ${task.id} AND created_at = ${task.createdAt}
887 AND project_id = ${task.projectId}
888 AND thread_id IS NULL AND thread_mode != 'new_thread'
889 `;
890 const shell = yield* threadManagement.getThreadShell(threadId);
891 if (shell === null || shell.deletedAt != null) yield* deleteForThread(threadId);
892 },
893 Effect.catchCause((cause) =>
894 Effect.logWarning("Could not bind a scheduled task to its first thread.", cause),
895 ),
896 );
897
898 // A dispatch is recorded once the message is accepted; the provider can still
899 // reject the turn after that, so follow the run and record its failure. A one-time
900 // task whose turn completed at or after its runAt is spent, so it is removed; its
901 // thread stays. A paused task always has a NULL next_run_at, so that alone proves nothing.
902 const recordRunOutcome = Effect.fn("ScheduledTaskService.recordRunOutcome")(function* (
903 taskId: ScheduledTaskId,
904 startedAtIso: string,
905 threadId: ThreadId,
906 messageId: MessageId,
907 ) {
908 const shell = yield* threadManagement.getThreadShell(threadId);
909 if (shell === null) return;
910 const thread = { projectId: shell.projectId, threadId };
911 const { runs } = yield* threadManagement.getProjectThreadRecords(thread, ["runs"]);
912 const dispatched = runs.find((run) => run.userMessageId === messageId);
913 if (dispatched === undefined) return;
914 const { run } = yield* threadManagement.waitForThread({
915 ...thread,
916 runId: dispatched.id,
917 timeoutMs: RUN_OUTCOME_WAIT_MS,
918 });
919 if (run === null || !ThreadManagementService.isTerminalRunStatus(run.status)) return;
920 if (run.status === "completed") {
921 const removed = yield* sql`
922 DELETE FROM scheduled_tasks
923 WHERE task_id = ${taskId} AND last_run_at = ${startedAtIso}
924 AND last_run_status = 'succeeded' AND next_run_at IS NULL
925 AND json_extract(schedule_json, '$.type') = 'once'
926 AND julianday(last_run_at) >= julianday(json_extract(schedule_json, '$.runAt'))
927 RETURNING task_id
928 `;
929 if (removed.length > 0) yield* notifyChanged;
930 return;
931 }
932 // Only the run's failed errors feed latestRootProviderFailure; a long-lived task
933 // thread holds tens of thousands of items.
934 const { turnItems, messages } = yield* threadManagement.getProjectThreadRecords(
935 thread,
936 ["turnItems", "messages"],
937 {
938 messageIds: [messageId],
939 turnItemRunId: run.id,
940 turnItemTypes: ["error"],
941 turnItemStatuses: ["failed"],
942 },
943 );
944 // A queued message promoted to a steer cancels its run and joins the active one.
945 if (
946 run.status === "cancelled" &&
947 messages.find((message) => message.id === messageId)?.runId !== run.id
948 )
949 return;
950 const error =
951 latestRootProviderFailure(run, turnItems)?.message ?? UNSUCCESSFUL_RUN_ERRORS[run.status];
952 const updated = yield* sql`
953 UPDATE scheduled_tasks
954 SET last_run_status = 'failed', last_run_error = ${error}
955 WHERE task_id = ${taskId} AND last_run_at = ${startedAtIso}
956 AND (last_run_status = 'succeeded'
957 OR (last_run_status = 'failed' AND last_run_error = ${HELD_OCCURRENCE_ERROR}))
958 RETURNING task_id
959 `;
960 if (updated.length > 0) yield* notifyChanged;
961 });
962
963 const runTask = Effect.fn("ScheduledTaskService.runTask")(function* (
964 task: ScheduledTask,
965 trigger: "scheduled" | "manual" | "webhook",
966 webhook?: {
967 readonly deliveryId: string;
968 readonly prompt: string;
969 readonly dispatched: Ref.Ref<boolean>;
970 },
971 ) {
972 // Read before reserving: nothing between the reservation and its release may fail.
973 const startedAt = yield* localNow;
974 const startedAtIso = iso(startedAt);
975 const run = Effect.gen(function* () {
976 // Re-read stale poll snapshots so deleted, paused or postponed tasks cannot fire.
977 const active = yield* findTask(task.id);
978 if (active === null) {
979 if (webhook !== undefined) {
980 return yield* new WebhookDeliverySkipped({
981 reason: "The task was deleted before this delivery ran.",
982 });
983 }
984 if (trigger !== "scheduled") {
985 return yield* taskError("Schedule task not found.", { taskId: task.id });
986 }
987 return { task, threadId: null, dispatchFailed: false };
988 }
989 if (
990 webhook !== undefined &&
991 (yield* sql`SELECT 1 FROM scheduled_task_webhook_pending WHERE delivery_id = ${webhook.deliveryId} AND state = 'completed'`.pipe(
992 NodeSqliteClient.readOnly,
993 )).length > 0
994 )
995 return {
996 task: { ...active, lastRunStatus: "succeeded" as const },
997 threadId: null,
998 dispatchFailed: false,
999 };
1000 const parsedNextRunAt =
1001 active.nextRunAt === null ? Option.none() : DateTime.make(active.nextRunAt);
1002 if (
1003 trigger === "scheduled" &&
1004 (!active.enabled ||
1005 Option.isNone(parsedNextRunAt) ||
1006 DateTime.toEpochMillis(parsedNextRunAt.value) > DateTime.toEpochMillis(startedAt))
1007 ) {
1008 return { task: active, threadId: null, dispatchFailed: false };
1009 }
1010 // A thread that stays busy would otherwise collect one copy of the
1011 // prompt per interval. A failed read falls through to the normal send.
1012 if (
1013 trigger === "scheduled" &&
1014 active.schedule.type !== "once" &&
1015 active.threadId !== null
1016 ) {
1017 const occurrence = yield* queuedOccurrence(
1018 active.id,
1019 ThreadId.make(active.threadId),
1020 ).pipe(Effect.orElseSucceed(() => null));
1021 if (occurrence !== null) {
1022 yield* rescheduleMissedRun(
1023 active,
1024 startedAt,
1025 "previous occurrence still queued",
1026 occurrence === "held" ? HELD_OCCURRENCE_ERROR : undefined,
1027 );
1028 return { task: active, threadId: null, dispatchFailed: false };
1029 }
1030 }
1031 // Reject queued deliveries after pause, same-id recreation or trigger changes.
1032 if (webhook !== undefined) {
1033 const reason =
1034 active.createdAt !== task.createdAt
1035 ? "The task was replaced before this delivery ran."
1036 : active.schedule.type !== "webhook"
1037 ? "The task's trigger changed before this delivery ran."
1038 : webhookPolicyBlock !== undefined
1039 ? webhookPolicyBlock
1040 : !active.enabled
1041 ? "The task was paused before this delivery ran."
1042 : null;
1043 if (reason !== null) return yield* new WebhookDeliverySkipped({ reason });
1044 }
1045
1046 if (webhook === undefined) yield* markRunning(active.id, startedAtIso);
1047 else
1048 yield* sql
1049 .withTransaction(
1050 Effect.gen(function* () {
1051 yield* markRunning(active.id, startedAtIso, webhook.deliveryId);
1052 // Recovery must distinguish this delivery from other windows waiting on the same task.
1053 yield* sql`UPDATE scheduled_task_webhook_pending SET state = 'running' WHERE delivery_id = ${webhook.deliveryId}`;
1054 }),
1055 )
1056 .pipe(
1057 Effect.mapError((cause) =>
1058 taskError("Could not start webhook run.", { taskId: task.id, cause }),
1059 ),
1060 );
1061
1062 yield* notifyChanged;
1063
1064 const fireKey =
1065 webhook === undefined
1066 ? `${active.id}:${DateTime.toEpochMillis(startedAt)}:${trigger}`
1067 : `${active.id}:webhook:${webhook.deliveryId}`;
1068 const commandId = CommandId.make(`scheduled-task:${fireKey}`);
1069 const messageId = MessageId.make(`scheduled-task-message:${fireKey}`);
1070 const prompt = webhook?.prompt ?? active.prompt;
1071 const threadMode = scheduledTaskThreadMode(active);
1072
1073 // A bound thread can move to another project (No project to a project) while
1074 // this fires. Follow it; one move needs at most two retries. Same ids replay safely.
1075 const sendBound = (
1076 threadId: ThreadId,
1077 projectId: ProjectId,
1078 retries: number,
1079 ): ReturnType<typeof threadManagement.sendToThread> =>
1080 threadManagement
1081 .sendToThread({
1082 projectId,
1083 commandId,
1084 threadId,
1085 messageId,
1086 scheduledTaskId: active.id,
1087 text: prompt,
1088 attachments: [],
1089 mode: "queue",
1090 ...(threadMode === "fresh_session" ? { freshContext: true } : {}),
1091 createdBy: active.createdBy,
1092 creationSource: active.creationSource,
1093 })
1094 .pipe(
1095 Effect.catchTags({
1096 ThreadManagementThreadNotFoundError: (error) =>
1097 retries === 0
1098 ? Effect.fail(error)
1099 : threadManagement.getThreadShell(threadId).pipe(
1100 Effect.orElseSucceed(() => null),
1101 Effect.flatMap((shell) =>
1102 shell === null || shell.projectId === projectId
1103 ? Effect.fail(error)
1104 : sendBound(threadId, shell.projectId, retries - 1),
1105 ),
1106 ),
1107 }),
1108 );
1109
1110 // Capture defects and interrupts too so dispatch failures still reach markCompleted.
1111 const result =
1112 active.threadId === null
1113 ? yield* Effect.exit(
1114 threadLaunch
1115 .launch({
1116 commandId,
1117 projectId: active.projectId,
1118 title: active.title,
1119 modelSelection: active.modelSelection,
1120 runtimeMode: active.runtimeMode,
1121 interactionMode: active.interactionMode,
1122 workspaceStrategy: active.workspaceStrategy,
1123 initialMessage: {
1124 messageId,
1125 scheduledTaskId: active.id,
1126 text: prompt,
1127 attachments: [],
1128 },
1129 createdBy: active.createdBy,
1130 creationSource: active.creationSource,
1131 })
1132 .pipe(
1133 Effect.map((launched) => launched.threadId),
1134 Effect.tap((threadId) =>
1135 threadMode === "new_thread" ? Effect.void : bindThread(active, threadId),
1136 ),
1137 ),
1138 )
1139 : yield* Effect.exit(
1140 // The stored selection only seeds fresh threads. A bound run
1141 // follows the thread's current provider, so it passes none.
1142 sendBound(ThreadId.make(active.threadId), active.projectId, 2).pipe(
1143 Effect.as(ThreadId.make(active.threadId)),
1144 ),
1145 );
1146
1147 const runSucceeded = result._tag === "Success";
1148 if (webhook !== undefined && runSucceeded) yield* Ref.set(webhook.dispatched, true);
1149 const completedAt = yield* localNow;
1150 const lastRunStatus = runSucceeded ? ("succeeded" as const) : ("failed" as const);
1151 const lastRunError = runSucceeded ? null : dispatchErrorMessage(result.cause);
1152 const threadId = runSucceeded ? result.value : null;
1153 // One transaction, so an edit cannot land between reading the schedule and
1154 // writing the next run computed from it. The origin is read first: it reads
1155 // files, and the transaction holds SQLite's writer lock.
1156 const origin = yield* readWebhookOrigin;
1157 const record = Effect.gen(function* () {
1158 const [row] = yield* getRows(task.id);
1159 const current = row === undefined ? null : yield* decodeRow(row, origin);
1160 const scheduleSource = current ?? task;
1161 const completed: ScheduledTask = {
1162 ...scheduleSource,
1163 updatedAt: iso(completedAt),
1164 lastRunAt: startedAtIso,
1165 // A manual run that starts before a one-time run must not use it up.
1166 nextRunAt: nextRunAt(
1167 scheduleSource,
1168 scheduleSource.schedule.type === "once" ? startedAt : completedAt,
1169 ),
1170 lastRunStatus,
1171 lastRunError,
1172 runCount: scheduleSource.runCount + 1,
1173 };
1174 if (current === null) return { recorded: completed, written: false };
1175 // startedAtIso in the guard ensures this writes only to the row this
1176 // run marked as running — a task deleted mid-run and recreated with
1177 // the same id (idempotent commandId replay) must not be stamped.
1178 const completion = markCompleted({
1179 id: task.id,
1180 completedAtIso: completed.updatedAt,
1181 nextRunAtIso: completed.nextRunAt,
1182 status: lastRunStatus,
1183 error: lastRunError,
1184 startedAtIso,
1185 });
1186 let written = false;
1187 if (webhook === undefined) written = yield* completion;
1188 else {
1189 const pending =
1190 yield* sql`SELECT 1 FROM scheduled_task_webhook_pending WHERE delivery_id = ${webhook.deliveryId}`;
1191 if (pending.length === 0) written = yield* completion;
1192 else {
1193 const counted =
1194 yield* sql`UPDATE scheduled_task_webhook_pending SET state = ${runSucceeded ? "completed" : "failed"}
1195 WHERE delivery_id = ${webhook.deliveryId} AND state NOT IN ('completed', 'failed') RETURNING delivery_id`;
1196 if (counted.length > 0) written = yield* completion;
1197 }
1198 }
1199 // A row recreated mid-run is not this run's, so report what is stored.
1200 return { recorded: written ? completed : current, written };
1201 });
1202 const { recorded, written } = yield* sql
1203 .withTransaction(record)
1204 .pipe(
1205 Effect.mapError((cause) =>
1206 isScheduledTaskError(cause)
1207 ? cause
1208 : taskError(
1209 webhook === undefined
1210 ? "Could not record schedule task run."
1211 : "Could not record webhook completion.",
1212 { taskId: task.id, cause },
1213 ),
1214 ),
1215 );
1216 if (written) yield* notifyChanged;
1217 if (written && threadId !== null) {
1218 yield* recordRunOutcome(active.id, startedAtIso, threadId, messageId).pipe(
1219 Effect.catchCause((cause) =>
1220 Effect.logWarning("Could not record scheduled run outcome", {
1221 taskId: active.id,
1222 cause,
1223 }),
1224 ),
1225 Effect.forkIn(serviceScope),
1226 );
1227 }
1228 // The stored row can belong to a replacement, so report this dispatch separately.
1229 return { task: recorded, threadId, dispatchFailed: !runSucceeded };
1230 }).pipe(
1231 Effect.onError((cause) =>
1232 releaseStuckRun(task, startedAtIso, errorMessage(cause), webhook?.deliveryId),
1233 ),
1234 );
1235
1236 // Reserve and release as one bracket so an interrupt cannot leave the task marked busy.
1237 return yield* Effect.acquireUseRelease(
1238 Ref.modify(activeRuns, (active) => {
1239 if (active.has(task.id)) return [false, active] as const;
1240 const next = new Set(active);
1241 next.add(task.id);
1242 return [true, next] as const;
1243 }),
1244 (reserved) =>
1245 reserved
1246 ? run
1247 : trigger !== "scheduled"
1248 ? Effect.fail(taskError("Schedule task is already running.", { taskId: task.id }))
1249 : Effect.succeed({ task, threadId: null, dispatchFailed: false }),
1250 (reserved) =>
1251 reserved
1252 ? Ref.update(activeRuns, (active) => {
1253 const next = new Set(active);
1254 next.delete(task.id);
1255 return next;
1256 })
1257 : Effect.void,
1258 );
1259 });
1260
1261 const rescheduleMissedRun = Effect.fn("ScheduledTaskService.rescheduleMissedRun")(function* (
1262 task: ScheduledTask,
1263 now: DateTime.DateTime,
1264 reason = "missed",
1265 runError?: string,
1266 ) {
1267 const next = nextRunAt(task, now);
1268 yield* Effect.logInfo("Skipping schedule task run", {
1269 taskId: task.id,
1270 reason,
1271 missedRunAt: task.nextRunAt,
1272 rescheduledTo: next,
1273 });
1274 // `task` can be a stale batch snapshot; a run or edit since then already moved the row on.
1275 const updated = yield* sql`
1276 UPDATE scheduled_tasks
1277 SET next_run_at = ${next},
1278 updated_at = ${iso(now)}
1279 ${
1280 runError === undefined
1281 ? sql``
1282 : sql`, last_run_status = CASE
1283 WHEN last_run_status IS ${task.lastRunStatus} AND last_run_error IS ${task.lastRunError}
1284 THEN 'failed' ELSE last_run_status END,
1285 last_run_error = CASE
1286 WHEN last_run_status IS ${task.lastRunStatus} AND last_run_error IS ${task.lastRunError}
1287 THEN ${runError} ELSE last_run_error END`
1288 }
1289 WHERE task_id = ${task.id}
1290 AND next_run_at IS ${task.nextRunAt}
1291 AND last_run_at IS ${task.lastRunAt}
1292 RETURNING task_id
1293 `.pipe(
1294 Effect.mapError((cause) =>
1295 taskError("Could not reschedule missed schedule task run.", { taskId: task.id, cause }),
1296 ),
1297 );
1298 if (updated.length > 0) yield* notifyChanged;
1299 });
1300
1301 const runDueTasks = Effect.fn("ScheduledTaskService.runDueTasks")(function* () {
1302 const now = yield* localNow;
1303 const nowMs = DateTime.toEpochMillis(now);
1304 if (yield* dueTasks.isIdle(nowMs)) return;
1305 const readVersion = yield* dueTasks.version;
1306 const [next] = yield* sql<{ next_run_at: string | null }>`
1307 SELECT MIN(next_run_at) AS next_run_at FROM scheduled_tasks
1308 WHERE enabled = 1 AND next_run_at IS NOT NULL AND last_run_status <> 'running'
1309 `.pipe(
1310 NodeSqliteClient.readOnly,
1311 Effect.mapError((cause) => taskError("Could not list schedule tasks.", { cause })),
1312 );
1313 // Compare as text like listDueTasks: expanded years (`+011533-…`) sort first but parse far ahead.
1314 const nextRunAt = next?.next_run_at ?? null;
1315 if (nextRunAt === null || nextRunAt > iso(now)) {
1316 const nextDueAt =
1317 nextRunAt === null
1318 ? Option.some(Infinity)
1319 : DateTime.make(nextRunAt).pipe(Option.map(DateTime.toEpochMillis));
1320 if (Option.isSome(nextDueAt) && nextDueAt.value > nowMs) {
1321 yield* dueTasks.record(readVersion, nextDueAt.value, nowMs);
1322 }
1323 return;
1324 }
1325 const tasks = yield* listDueTasks(now).pipe(
1326 Effect.mapError((cause) => taskError("Could not list schedule tasks.", { cause })),
1327 );
1328 const due = tasks.flatMap((task) =>
1329 task.nextRunAt === null ? [] : [{ task, dueAt: DateTime.makeUnsafe(task.nextRunAt) }],
1330 );
1331 yield* Effect.forEach(
1332 due,
1333 ({ task, dueAt }) =>
1334 Effect.suspend(() =>
1335 isMissedFixedTimeRun(task.schedule, dueAt, now)
1336 ? rescheduleMissedRun(
1337 task,
1338 now,
1339 "missed",
1340 "Missed a scheduled run because the server was not running at that time.",
1341 )
1342 : runTask(task, "scheduled"),
1343 ).pipe(
1344 Effect.catch((cause) =>
1345 Effect.logWarning("Scheduled task run failed", { taskId: task.id, cause }),
1346 ),
1347 ),
1348 { concurrency: 1, discard: true },
1349 );
1350 });
1351
1352 // Recover crash-left running rows that the due-task filter would otherwise skip forever.
1353 yield* Effect.gen(function* () {
1354 const rows = yield* selectAllRows();
1355 const stuck = rows.filter((row) => row.last_run_status === "running");
1356 if (stuck.length === 0) return;
1357 const now = yield* localNow;
1358 yield* Effect.forEach(
1359 stuck,
1360 (row) =>
1361 Effect.gen(function* () {
1362 const decoded = yield* Effect.result(decodeRow(row));
1363 if (Result.isSuccess(decoded)) {
1364 yield* sql`
1365 UPDATE scheduled_tasks
1366 SET last_run_status = 'failed',
1367 last_run_error = 'Run was interrupted by a server restart.',
1368 next_run_at = ${nextRunAfterRun(decoded.success, now)},
1369 updated_at = ${iso(now)},
1370 run_count = run_count + CASE WHEN EXISTS (
1371 SELECT 1 FROM scheduled_task_webhook_pending
1372 WHERE task_id IS ${row.task_id} AND state = 'running'
1373 AND delivery_id = scheduled_tasks.last_run_webhook_delivery_id
1374 ) THEN 0 ELSE 1 END
1375 WHERE task_id IS ${row.task_id} AND last_run_status = 'running'
1376 `;
1377 return;
1378 }
1379 yield* Effect.logWarning(
1380 "Recovering undecodable schedule task row without rescheduling",
1381 { taskId: row.task_id, cause: decoded.failure },
1382 );
1383 yield* sql`
1384 UPDATE scheduled_tasks
1385 SET last_run_status = 'failed',
1386 last_run_error = 'Run was interrupted by a server restart.',
1387 updated_at = ${iso(now)},
1388 run_count = run_count + CASE WHEN EXISTS (
1389 SELECT 1 FROM scheduled_task_webhook_pending
1390 WHERE task_id IS ${row.task_id} AND state = 'running'
1391 AND delivery_id = scheduled_tasks.last_run_webhook_delivery_id
1392 ) THEN 0 ELSE 1 END
1393 WHERE task_id IS ${row.task_id} AND last_run_status = 'running'
1394 `;
1395 }),
1396 { concurrency: 1, discard: true },
1397 );
1398 yield* notifyChanged;
1399 }).pipe(
1400 Effect.catch((cause) =>
1401 Effect.logWarning("Could not reset interrupted schedule task runs", { cause }),
1402 ),
1403 );
1404
1405 yield* scheduler.register("scheduled-tasks", runDueTasks());
1406
1407 const list: ScheduledTaskService["Service"]["list"] = () =>
1408 listRows().pipe(
1409 Effect.map((tasks) => ({
1410 tasks,
1411 timeZone: DateTime.zoneToString(DateTime.zoneMakeLocal()),
1412 })),
1413 Effect.mapError((cause) => taskError("Could not list schedule tasks.", { cause })),
1414 );
1415
1416 const subscribeList: ScheduledTaskService["Service"]["subscribeList"] = () =>
1417 Stream.unwrap(
1418 Effect.gen(function* () {
1419 // Subscribe before reading to buffer concurrent changes.
1420 const subscription = yield* PubSub.subscribe(changesPubSub);
1421 return Stream.concat(
1422 Stream.fromEffect(list()),
1423 Stream.fromSubscription(subscription).pipe(Stream.mapEffect(() => list())),
1424 );
1425 }),
1426 );
1427
1428 const requireDestinationThread = Effect.fn("ScheduledTaskService.requireDestinationThread")(
1429 function* (threadId: ThreadId, projectId: ProjectId, taskId: ScheduledTaskId) {
1430 const thread = yield* threadManagement
1431 .getThreadShell(threadId)
1432 .pipe(
1433 Effect.mapError((cause) =>
1434 taskError("Could not read the destination thread.", { taskId, cause }),
1435 ),
1436 );
1437 if (thread === null || thread.deletedAt !== null) {
1438 return yield* taskError("That thread no longer exists.", { taskId });
1439 }
1440 if (thread.archivedAt !== null) {
1441 return yield* taskError(
1442 "That thread is archived. Unarchive it or pick another destination.",
1443 { taskId },
1444 );
1445 }
1446 if (thread.projectId !== projectId) {
1447 return yield* taskError("That thread belongs to a different project.", { taskId });
1448 }
1449 },
1450 );
1451
1452 const upsert: ScheduledTaskService["Service"]["upsert"] = Effect.fn(
1453 "ScheduledTaskService.upsert",
1454 )(function* (input) {
1455 // A webhook task the policy blocks can still be edited while it is off.
1456 if (input.enabled && input.schedule.type === "webhook" && webhookPolicyBlock !== undefined) {
1457 return yield* taskError(
1458 webhookPolicyBlock,
1459 input.id === undefined ? {} : { taskId: input.id },
1460 );
1461 }
1462 const now = yield* localNow;
1463 const uuid =
1464 input.commandId === undefined
1465 ? yield* random
1466 .uuidV4()
1467 .pipe(
1468 Effect.mapError((cause) =>
1469 taskError("Could not generate schedule task id.", { cause }),
1470 ),
1471 )
1472 : null;
1473 const id =
1474 input.id ??
1475 ScheduledTaskId.make(
1476 input.commandId ? `scheduled-task:${input.commandId}` : `scheduled-task:${uuid}`,
1477 );
1478 // Look up by the *resolved* id so idempotent creates (commandId replays)
1479 // keep their run history, and so real load failures propagate instead
1480 // of silently resetting an existing row.
1481 const existingTask = yield* findTask(id);
1482 const threadMode = resolveThreadMode(input, existingTask);
1483 const keptThreadId =
1484 input.threadId ?? (input.unbindThread === true ? null : (existingTask?.threadId ?? null));
1485 // A bound thread belongs to its project, so moving the task leaves it for a new one.
1486 const movedOffThread =
1487 existingTask !== null &&
1488 existingTask.projectId !== input.projectId &&
1489 keptThreadId === existingTask.threadId;
1490 const threadId = threadMode === "new_thread" || movedOffThread ? null : keptThreadId;
1491 // An unchanged destination is not rechecked, so a task on an archived thread stays editable.
1492 if (threadId !== null && threadId !== existingTask?.threadId) {
1493 yield* requireDestinationThread(threadId, input.projectId, id);
1494 }
1495 // Refused on save rather than at run time; a disabled task can still be saved, so one
1496 // stored above the ceiling can be turned off before its mode is lowered. A task bound to a
1497 // thread runs in that thread's mode, which the orchestrator checks instead. Checked before
1498 // the save consumes a webhook secret.
1499 const modeDenial =
1500 input.enabled && threadId === null
1501 ? managedRuntimeModeDenial(managedPolicy.summary, input.runtimeMode)
1502 : undefined;
1503 if (modeDenial !== undefined) return yield* taskError(modeDenial, { taskId: id });
1504 // Keep the existing next_run_at when the schedule itself is untouched:
1505 // editing a title or prompt must not postpone (or resurrect) a due
1506 // run — only schedule/enabled changes restart the clock.
1507 const schedule: ScheduledTask["schedule"] =
1508 input.schedule.type === "webhook"
1509 ? {
1510 type: "webhook",
1511 signature:
1512 input.schedule.signature == null
1513 ? null
1514 : {
1515 header: input.schedule.signature.header.toLowerCase(),
1516 encoding: input.schedule.signature.encoding,
1517 prefix: input.schedule.signature.prefix,
1518 },
1519 maxDeliveryAgeMinutes: input.schedule.maxDeliveryAgeMinutes ?? null,
1520 ...(input.schedule.dedupe === undefined
1521 ? existingTask?.schedule.type === "webhook" &&
1522 existingTask.schedule.dedupe !== undefined
1523 ? { dedupe: existingTask.schedule.dedupe }
1524 : {}
1525 : { dedupe: input.schedule.dedupe }),
1526 ...webhookFilters(
1527 // Omitted keeps the stored filters, so a client that predates
1528 // them can re-save a task without clearing them.
1529 input.schedule.filters ??
1530 (existingTask?.schedule.type === "webhook"
1531 ? existingTask.schedule.filters
1532 : undefined),
1533 ),
1534 }
1535 : input.schedule.type === "once"
1536 ? { type: "once", runAt: iso(DateTime.makeUnsafe(input.schedule.runAt)) }
1537 : input.schedule;
1538 const webhook =
1539 input.schedule.type === "webhook"
1540 ? yield* Effect.gen(function* () {
1541 const existing = existingTask === null ? null : yield* findWebhookCredentials(id);
1542 const token = existing?.token ?? (yield* newWebhookToken);
1543 const signature = input.schedule.type === "webhook" ? input.schedule.signature : null;
1544 // A secretRef is a value the user entered for an agent; this
1545 // save consumes it, so it cannot be used again. A replay of a
1546 // save that already used it keeps the secret that save stored.
1547 const replay = input.commandId !== undefined && existing?.secret != null;
1548 const fromRef =
1549 signature?.secretRef === undefined
1550 ? undefined
1551 : yield* secretRequests
1552 .consume({ ref: signature.secretRef, projectId: input.projectId })
1553 .pipe(
1554 Effect.catch((error) =>
1555 replay
1556 ? Effect.void
1557 : Effect.fail(taskError(error.message, { taskId: id })),
1558 ),
1559 );
1560 // A ref, when given, is the only source: a plain secret sent
1561 // alongside it must not replace what a replayed save stored.
1562 const provided = signature?.secretRef === undefined ? signature?.secret : fromRef;
1563 const secret = signature == null ? null : (provided ?? existing?.secret ?? null);
1564 const secretChanged =
1565 signature == null || provided !== undefined || existing === null;
1566 if (signature != null && secret === null) {
1567 return yield* taskError("A webhook signature check needs a signing secret.", {
1568 taskId: id,
1569 });
1570 }
1571 return { token, secret, secretChanged };
1572 })
1573 : { token: null, secret: null, secretChanged: true };
1574 // A folder that is not a Git repository, Scratch included, cannot hold a worktree.
1575 const workspaceStrategy: ScheduledTask["workspaceStrategy"] =
1576 input.workspaceStrategy.type === "worktree" &&
1577 ((yield* managedFolders.isScratchProject(input.projectId)) ||
1578 (yield* threadLaunch.isNonGitProject(input.projectId)))
1579 ? { type: "root" }
1580 : input.workspaceStrategy;
1581 const scheduleUnchanged =
1582 existingTask !== null &&
1583 existingTask.enabled === input.enabled &&
1584 isSameSchedule(existingTask.schedule, schedule);
1585 if (!scheduleUnchanged)
1586 yield* requireFutureOnce({ enabled: input.enabled, schedule }, now, id);
1587 const task: ScheduledTask = {
1588 id,
1589 title: input.title,
1590 prompt: input.prompt,
1591 enabled: input.enabled,
1592 schedule,
1593 projectId: input.projectId,
1594 threadId,
1595 threadMode,
1596 workspaceStrategy,
1597 modelSelection: input.modelSelection,
1598 runtimeMode: input.runtimeMode,
1599 interactionMode: input.interactionMode,
1600 createdBy: existingTask?.createdBy ?? input.createdBy ?? "user",
1601 creationSource: input.creationSource ?? "web",
1602 createdAt: existingTask?.createdAt ?? iso(now),
1603 updatedAt: iso(now),
1604 nextRunAt: scheduleUnchanged
1605 ? existingTask.nextRunAt
1606 : nextRunAt({ enabled: input.enabled, schedule }, now),
1607 lastRunAt: existingTask?.lastRunAt ?? null,
1608 lastRunStatus: existingTask?.lastRunStatus ?? "never",
1609 lastRunError: existingTask?.lastRunError ?? null,
1610 runCount: existingTask?.runCount ?? 0,
1611 };
1612 yield* saveTask(
1613 task,
1614 input.requireExisting === true,
1615 scheduleUnchanged ? existingTask.schedule : null,
1616 threadId === null && input.unbindThread === true ? (existingTask?.threadId ?? null) : null,
1617 webhook,
1618 );
1619 yield* notifyChanged;
1620 return { task: (yield* findTask(id)) ?? task };
1621 });
1622
1623 const setEnabled: ScheduledTaskService["Service"]["setEnabled"] = Effect.fn(
1624 "ScheduledTaskService.setEnabled",
1625 )(function* (input) {
1626 const existing = yield* loadTask(input.id);
1627 if (existing.enabled === input.enabled) return { task: existing };
1628 if (
1629 input.enabled &&
1630 existing.schedule.type === "webhook" &&
1631 webhookPolicyBlock !== undefined
1632 ) {
1633 return yield* taskError(webhookPolicyBlock, { taskId: input.id });
1634 }
1635 const modeDenial =
1636 input.enabled && existing.threadId === null
1637 ? managedRuntimeModeDenial(managedPolicy.summary, existing.runtimeMode)
1638 : undefined;
1639 if (modeDenial !== undefined) {
1640 return yield* taskError(modeDenial, { taskId: input.id });
1641 }
1642 const now = yield* localNow;
1643 yield* requireFutureOnce(
1644 { enabled: input.enabled, schedule: existing.schedule },
1645 now,
1646 input.id,
1647 );
1648 const next = nextRunAt({ enabled: input.enabled, schedule: existing.schedule }, now);
1649 // RETURNING so a task deleted between the load and this UPDATE is a
1650 // visible not-found error, not a false success.
1651 const updated = yield* sql<{ task_id: string }>`
1652 UPDATE scheduled_tasks
1653 SET enabled = ${input.enabled ? 1 : 0},
1654 next_run_at = ${next},
1655 updated_at = ${iso(now)}
1656 WHERE task_id = ${input.id}
1657 RETURNING task_id
1658 `.pipe(
1659 Effect.mapError((cause) =>
1660 taskError("Could not update schedule task.", { taskId: input.id, cause }),
1661 ),
1662 );
1663 if (updated.length === 0) {
1664 return yield* taskError("Schedule task not found.", { taskId: input.id });
1665 }
1666 yield* notifyChanged;
1667 return {
1668 task: { ...existing, enabled: input.enabled, nextRunAt: next, updatedAt: iso(now) },
1669 };
1670 });
1671
1672 const forgetWebhookTask = (id: ScheduledTaskId) =>
1673 Effect.all(
1674 [
1675 Ref.update(webhookRateWindows, (windows) => {
1676 const next = new Map(windows);
1677 next.delete(id);
1678 return next;
1679 }),
1680 Ref.update(webhookRejectedRateWindows, (windows) => {
1681 const next = new Map(windows);
1682 next.delete(id);
1683 return next;
1684 }),
1685 Ref.update(webhookDedupeLogWindows, (windows) => {
1686 const next = new Map(windows);
1687 next.delete(id);
1688 return next;
1689 }),
1690 Ref.update(webhookFilteredLogWindows, (windows) => {
1691 const next = new Map(windows);
1692 next.delete(id);
1693 return next;
1694 }),
1695 Ref.update(webhookPermits, (permits) => {
1696 const next = new Map(permits);
1697 next.delete(id);
1698 return next;
1699 }),
1700 ],
1701 { discard: true },
1702 );
1703
1704 const deleteTask: ScheduledTaskService["Service"]["delete"] = (input) =>
1705 deleteRow(input.id).pipe(
1706 Effect.andThen(forgetWebhookTask(input.id)),
1707 Effect.andThen(notifyChanged),
1708 Effect.as({ id: input.id }),
1709 );
1710
1711 const deleteForThread = Effect.fn("ScheduledTaskService.deleteForThread")(function* (
1712 threadId: ThreadId,
1713 ) {
1714 // Check the current binding in the transaction: an automation may have
1715 // been unbound or moved since thread deletion queued this cleanup.
1716 const removed = yield* sql
1717 .withTransaction(
1718 Effect.gen(function* () {
1719 yield* sql`
1720 DELETE FROM scheduled_task_webhook_deliveries
1721 WHERE task_id IN (SELECT task_id FROM scheduled_tasks WHERE thread_id = ${threadId})
1722 `;
1723 return yield* sql<{ task_id: ScheduledTaskId }>`
1724 DELETE FROM scheduled_tasks WHERE thread_id = ${threadId} RETURNING task_id
1725 `;
1726 }),
1727 )
1728 .pipe(
1729 Effect.mapError((cause) =>
1730 taskError("Could not delete thread's schedule tasks.", { cause }),
1731 ),
1732 );
1733 yield* Effect.forEach(removed, ({ task_id }) => forgetWebhookTask(task_id), {
1734 discard: true,
1735 });
1736 if (removed.length > 0) yield* notifyChanged;
1737 });
1738
1739 const moveThreadTasks: ScheduledTaskService["Service"]["moveThreadTasks"] = Effect.fn(
1740 "ScheduledTaskService.moveThreadTasks",
1741 )(function* (input) {
1742 if (input.threadIds.length === 0) return;
1743 const moved = yield* sql<{ task_id: ScheduledTaskId }>`
1744 UPDATE scheduled_tasks SET project_id = ${input.projectId}
1745 WHERE project_id != ${input.projectId} AND thread_id IN ${sql.in(input.threadIds)}
1746 RETURNING task_id
1747 `.pipe(
1748 Effect.mapError((cause) =>
1749 taskError("Could not move the thread's schedule tasks.", { cause }),
1750 ),
1751 );
1752 if (moved.length > 0) yield* notifyChanged;
1753 });
1754
1755 const runNow: ScheduledTaskService["Service"]["runNow"] = Effect.fn(
1756 "ScheduledTaskService.runNow",
1757 )(function* (input: ScheduledTaskRunNowInput) {
1758 const task = yield* loadTask(input.id);
1759 if (task.schedule.type === "webhook") {
1760 return yield* taskError("Webhook tasks run when their URL receives a request.", {
1761 taskId: input.id,
1762 });
1763 }
1764 return yield* runTask(task, "manual").pipe(
1765 Effect.mapError((cause) =>
1766 isScheduledTaskError(cause)
1767 ? cause
1768 : taskError("Could not run schedule task.", { taskId: input.id, cause }),
1769 ),
1770 );
1771 });
1772
1773 const rotateWebhookToken: ScheduledTaskService["Service"]["rotateWebhookToken"] = Effect.fn(
1774 "ScheduledTaskService.rotateWebhookToken",
1775 )(function* (input) {
1776 const task = yield* loadTask(input.id);
1777 if (task.schedule.type !== "webhook") {
1778 return yield* taskError("Only webhook tasks have a URL token.", { taskId: input.id });
1779 }
1780 const token = yield* newWebhookToken;
1781 const now = yield* localNow;
1782 // Matching created_at keeps a rotation from landing on a task deleted
1783 // and recreated under the same id since it was loaded.
1784 const updated = yield* sql<{ task_id: string }>`
1785 UPDATE scheduled_tasks
1786 SET webhook_token = ${token}, updated_at = ${iso(now)}
1787 WHERE task_id = ${input.id} AND created_at = ${task.createdAt}
1788 RETURNING task_id
1789 `.pipe(
1790 Effect.mapError((cause) =>
1791 taskError("Could not rotate webhook token.", { taskId: input.id, cause }),
1792 ),
1793 );
1794 if (updated.length === 0) {
1795 return yield* taskError("Schedule task was deleted or replaced.", { taskId: input.id });
1796 }
1797 yield* notifyChanged;
1798 return { task: yield* loadTask(input.id) };
1799 });
1800
1801 const deliveryHeaders = (row: WebhookDeliveryRow): Readonly<Record<string, string>> =>
1802 Option.getOrElse(decodeHeadersJson(row.headers_json), () => ({}));
1803 const decodeDeliverySummary = (row: WebhookDeliveryRow) => ({
1804 id: ScheduledTaskWebhookDeliveryId.make(row.delivery_id),
1805 taskId: ScheduledTaskId.make(row.task_id),
1806 receivedAt: row.received_at,
1807 method: row.method,
1808 contentType: deliveryHeaders(row)["content-type"] ?? null,
1809 bodyBytes: row.body_bytes,
1810 outcome: row.outcome as ScheduledTaskWebhookDeliveryOutcome,
1811 signatureVerified: row.signature_verified === 1,
1812 pendingWindow: row.outcome === "accepted" && row.pending_window === 1,
1813 missingFields: Option.getOrElse(decodeMissingFieldsJson(row.missing_fields_json), () => []),
1814 error: row.error,
1815 });
1816
1817 const listWebhookDeliveries: ScheduledTaskService["Service"]["listWebhookDeliveries"] = (
1818 input,
1819 ) =>
1820 sql<WebhookDeliveryRow>`
1821 SELECT d.*, EXISTS (SELECT 1 FROM scheduled_task_webhook_pending p
1822 WHERE p.delivery_id = d.delivery_id AND p.state IN ('pending', 'dispatching')) AS pending_window
1823 FROM scheduled_task_webhook_deliveries d WHERE task_id = ${input.id}
1824 ORDER BY received_at DESC, rowid DESC
1825 `.pipe(
1826 NodeSqliteClient.readOnly,
1827 Effect.map((rows) => ({
1828 deliveries: rows
1829 .filter(
1830 (row) => input.acceptAnyOutcome === true || LEGACY_DELIVERY_OUTCOMES.has(row.outcome),
1831 )
1832 .map(decodeDeliverySummary),
1833 })),
1834 Effect.mapError((cause) =>
1835 taskError("Could not list webhook deliveries.", { taskId: input.id, cause }),
1836 ),
1837 );
1838
1839 const getWebhookDelivery: ScheduledTaskService["Service"]["getWebhookDelivery"] = Effect.fn(
1840 "ScheduledTaskService.getWebhookDelivery",
1841 )(function* (input) {
1842 const rows = yield* sql<WebhookDeliveryRow>`
1843 SELECT d.*, EXISTS (SELECT 1 FROM scheduled_task_webhook_pending p
1844 WHERE p.delivery_id = d.delivery_id AND p.state IN ('pending', 'dispatching')) AS pending_window
1845 FROM scheduled_task_webhook_deliveries d WHERE task_id = ${input.id} AND delivery_id = ${input.deliveryId}
1846 `.pipe(
1847 NodeSqliteClient.readOnly,
1848 Effect.mapError((cause) =>
1849 taskError("Could not load webhook delivery.", { taskId: input.id, cause }),
1850 ),
1851 );
1852 const row = rows[0];
1853 if (
1854 row === undefined ||
1855 (input.acceptAnyOutcome !== true && !LEGACY_DELIVERY_OUTCOMES.has(row.outcome))
1856 ) {
1857 return yield* taskError("Webhook delivery not found.", { taskId: input.id });
1858 }
1859 return {
1860 delivery: {
1861 ...decodeDeliverySummary(row),
1862 query: row.query,
1863 headers: deliveryHeaders(row),
1864 body: row.body,
1865 bodyTruncated: row.body_truncated === 1,
1866 renderedPrompt: row.rendered_prompt,
1867 },
1868 };
1869 });
1870
1871 const deliveryRequestBody = (request: WebhookTriggerRequest) => {
1872 const truncated = request.body.byteLength > WEBHOOK_DELIVERY_LOG_BODY_LIMIT;
1873 return {
1874 truncated,
1875 // Streaming decode drops a character the byte limit split, instead of ending in U+FFFD.
1876 body: truncated
1877 ? new TextDecoder().decode(request.body.subarray(0, WEBHOOK_DELIVERY_LOG_BODY_LIMIT), {
1878 stream: true,
1879 })
1880 : request.bodyText,
1881 };
1882 };
1883 const replaceDeliveryRequest = (deliveryId: string, request: WebhookTriggerRequest) => {
1884 const logged = deliveryRequestBody(request);
1885 return sql`UPDATE scheduled_task_webhook_deliveries SET
1886 method = ${request.method}, query = ${redactQuery(request.query)},
1887 headers_json = ${encodeHeadersJson(redactHeaders(request.headers))}, body = ${logged.body},
1888 body_bytes = ${request.body.byteLength}, body_truncated = ${logged.truncated ? 1 : 0}
1889 WHERE delivery_id = ${deliveryId}`.pipe(Effect.asVoid, Effect.mapError(dedupeError));
1890 };
1891
1892 // Each suppression bucket retains its own rows, never real run history.
1893 const deliveryBucket = (outcome: string, error: string | undefined) =>
1894 outcome === "filtered"
1895 ? "filtered"
1896 : outcome === "duplicate" ||
1897 outcome === "coalesced" ||
1898 outcome === "rate_limited" ||
1899 error === PROMPT_TOO_LONG
1900 ? "dedupe"
1901 : "main";
1902 const pruneDeliveries = (
1903 taskId: string,
1904 retentionBucket: ReturnType<typeof deliveryBucket>,
1905 ) => {
1906 const rowBucket = sql.literal(`CASE WHEN outcome = 'filtered' THEN 'filtered'
1907 WHEN outcome IN ('duplicate', 'coalesced', 'rate_limited') OR error = 'The filled-in prompt is too long.' THEN 'dedupe'
1908 ELSE 'main' END`);
1909 return sql`
1910 DELETE FROM scheduled_task_webhook_deliveries
1911 WHERE task_id = ${taskId} AND ${rowBucket} = ${retentionBucket}
1912 AND delivery_id NOT IN (
1913 SELECT delivery_id FROM scheduled_task_webhook_deliveries
1914 WHERE task_id = ${taskId} AND ${rowBucket} = ${retentionBucket}
1915 ORDER BY received_at DESC, rowid DESC
1916 LIMIT ${retentionBucket === "main" ? WEBHOOK_DELIVERY_RETENTION : retentionBucket === "filtered" ? WEBHOOK_FILTERED_DELIVERY_RETENTION : WEBHOOK_DEDUPE_DELIVERY_RETENTION}
1917 )
1918 `;
1919 };
1920
1921 const recordDelivery = (input: {
1922 readonly id: string;
1923 readonly taskId: ScheduledTaskId;
1924 readonly receivedAt: string;
1925 readonly request: WebhookTriggerRequest;
1926 readonly outcome: ScheduledTaskWebhookDeliveryOutcome;
1927 readonly signatureVerified: boolean;
1928 readonly missing: ReadonlyArray<string>;
1929 readonly renderedPrompt: string | null;
1930 readonly error?: string;
1931 }) => {
1932 const { truncated, body: loggedBody } = deliveryRequestBody(input.request);
1933 return sql
1934 .withTransaction(
1935 Effect.gen(function* () {
1936 // Require the task to exist so deletes cannot leave deliveries for a recreated task.
1937 yield* sql`
1938 INSERT INTO scheduled_task_webhook_deliveries (
1939 delivery_id, task_id, received_at, method, query, headers_json, body,
1940 body_bytes, body_truncated, outcome, signature_verified,
1941 missing_fields_json, rendered_prompt, error
1942 )
1943 SELECT
1944 ${input.id}, ${input.taskId}, ${input.receivedAt}, ${input.request.method},
1945 ${redactQuery(input.request.query)},
1946 ${encodeHeadersJson(redactHeaders(input.request.headers))},
1947 ${loggedBody},
1948 ${input.request.body.byteLength}, ${truncated ? 1 : 0}, ${input.outcome},
1949 ${input.signatureVerified ? 1 : 0}, ${encodeMissingFieldsJson(input.missing)},
1950 ${input.renderedPrompt?.slice(0, WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT) ?? null},
1951 ${input.error ?? null}
1952 WHERE EXISTS (SELECT 1 FROM scheduled_tasks WHERE task_id = ${input.taskId})
1953 -- A held request retried after a rate limit reuses its relay
1954 -- delivery id; the newer attempt replaces the logged one.
1955 ON CONFLICT (delivery_id) DO UPDATE SET
1956 outcome = excluded.outcome,
1957 signature_verified = excluded.signature_verified,
1958 missing_fields_json = excluded.missing_fields_json,
1959 rendered_prompt = excluded.rendered_prompt,
1960 error = excluded.error
1961 `;
1962 yield* pruneDeliveries(input.taskId, deliveryBucket(input.outcome, input.error));
1963 }),
1964 )
1965 .pipe(
1966 Effect.mapError((cause) =>
1967 taskError("Could not record webhook delivery.", { taskId: input.taskId, cause }),
1968 ),
1969 );
1970 };
1971
1972 const markDeliveryFailed = (deliveryId: string, message: string) =>
1973 sql
1974 .withTransaction(
1975 Effect.gen(function* () {
1976 const [updated] = yield* sql<{ task_id: string }>`
1977 UPDATE scheduled_task_webhook_deliveries
1978 SET outcome = 'dispatch_failed', error = ${message}
1979 WHERE delivery_id = ${deliveryId}
1980 RETURNING task_id
1981 `;
1982 // An oversized prompt moves the row into the suppression bucket.
1983 if (updated !== undefined)
1984 yield* pruneDeliveries(updated.task_id, deliveryBucket("dispatch_failed", message));
1985 }),
1986 )
1987 .pipe(Effect.ignore);
1988
1989 /** Sliding one-minute window for deliveries that can start work. */
1990 const takeRateSlot = (
1991 id: ScheduledTaskId,
1992 nowMs: number,
1993 rateWindows = webhookRateWindows,
1994 reservation = { at: nowMs },
1995 ) =>
1996 Ref.modify(
1997 rateWindows,
1998 (
1999 windows,
2000 ): readonly [
2001 "allowed" | "first_rejected" | "rejected",
2002 ReadonlyMap<ScheduledTaskId, RateWindow>,
2003 ] => {
2004 const current = windows.get(id);
2005 const recent = (current?.accepted ?? []).filter((entry) => nowMs - entry.at < 60_000);
2006 if (recent.length >= WEBHOOK_RATE_LIMIT_PER_MINUTE) {
2007 const first = current?.rejectedLogged !== true;
2008 return [
2009 first ? "first_rejected" : "rejected",
2010 new Map(windows).set(id, { accepted: recent, rejectedLogged: true }),
2011 ];
2012 }
2013 return [
2014 "allowed",
2015 new Map(windows).set(id, { accepted: [...recent, reservation], rejectedLogged: false }),
2016 ];
2017 },
2018 );
2019
2020 const takeDedupeLogSlot = (id: ScheduledTaskId, nowMs: number) =>
2021 Ref.modify(webhookDedupeLogWindows, (windows) => {
2022 const recent = (windows.get(id) ?? []).filter((at) => nowMs - at < 60_000);
2023 return recent.length >= WEBHOOK_DEDUPE_LOG_PER_MINUTE
2024 ? ([false, windows] as const)
2025 : ([true, new Map(windows).set(id, [...recent, nowMs])] as const);
2026 });
2027 const releaseReplay = (deliveryId: string) =>
2028 releaseWebhookReplay(deliveryId).pipe(
2029 Effect.provideService(SqlClient.SqlClient, sql),
2030 Effect.mapError(dedupeError),
2031 );
2032 /** Sliding one-minute window of filtered requests written to the delivery log. */
2033 const takeFilteredLogSlot = (id: ScheduledTaskId, nowMs: number) =>
2034 Ref.modify(
2035 webhookFilteredLogWindows,
2036 (windows): readonly [boolean, ReadonlyMap<ScheduledTaskId, ReadonlyArray<number>>] => {
2037 const recent = (windows.get(id) ?? []).filter((at) => nowMs - at < 60_000);
2038 return recent.length >= WEBHOOK_FILTERED_LOG_PER_MINUTE
2039 ? [false, windows]
2040 : [true, new Map(windows).set(id, [...recent, nowMs])];
2041 },
2042 );
2043
2044 const webhookPermit = Effect.fn("ScheduledTaskService.webhookPermit")(function* (
2045 id: ScheduledTaskId,
2046 ) {
2047 const existing = (yield* Ref.get(webhookPermits)).get(id);
2048 if (existing !== undefined) return existing;
2049 const created = yield* Semaphore.make(1);
2050 return yield* Ref.modify(webhookPermits, (permits) => {
2051 const raced = permits.get(id);
2052 return raced === undefined
2053 ? [created, new Map(permits).set(id, created)]
2054 : [raced, permits];
2055 });
2056 });
2057
2058 const releaseFailedDelivery = Effect.fn("ScheduledTaskService.releaseFailedDelivery")(
2059 function* (deliveryId: string) {
2060 yield* releaseReplay(deliveryId);
2061 yield* sql`DELETE FROM scheduled_task_webhook_relay_deliveries WHERE relay_delivery_id IN (
2062 SELECT substr(value, ${"delivery:relay:".length + 1}) FROM scheduled_task_webhook_pending, json_each(delivery_ids_json)
2063 WHERE delivery_id = ${deliveryId} AND value LIKE 'delivery:relay:%'
2064 )`.pipe(Effect.mapError(dedupeError));
2065 if (deliveryId.startsWith("delivery:relay:"))
2066 yield* sql`DELETE FROM scheduled_task_webhook_relay_deliveries WHERE relay_delivery_id = ${deliveryId.slice("delivery:relay:".length)}`.pipe(
2067 Effect.mapError(dedupeError),
2068 );
2069 },
2070 (effect) => sql.withTransaction(effect).pipe(Effect.mapError(dedupeError)),
2071 );
2072
2073 const runWebhookDelivery = Effect.fn("ScheduledTaskService.runWebhookDelivery")(function* (
2074 task: ScheduledTask,
2075 deliveryId: string,
2076 prompt: string,
2077 ) {
2078 const runOutcome = (outcome: "started" | "skipped" | "failed") =>
2079 Effect.all([
2080 Effect.annotateCurrentSpan({ "scheduled_task.webhook.run_outcome": outcome }),
2081 Metrics.increment(Metrics.webhookRunsTotal, { outcome }),
2082 ]);
2083 const dispatched = yield* Ref.make(false);
2084 return yield* runTask(task, "webhook", { deliveryId, prompt, dispatched }).pipe(
2085 Effect.flatMap(({ dispatchFailed }) =>
2086 dispatchFailed
2087 ? runOutcome("failed").pipe(
2088 Effect.andThen(markDeliveryFailed(deliveryId, "The run failed to start.")),
2089 Effect.andThen(releaseFailedDelivery(deliveryId)),
2090 )
2091 : runOutcome("started"),
2092 ),
2093 Effect.catchTags({
2094 WebhookDeliverySkipped: (skipped) =>
2095 runOutcome("skipped").pipe(
2096 Effect.andThen(markDeliveryFailed(deliveryId, skipped.reason)),
2097 Effect.andThen(releaseFailedDelivery(deliveryId)),
2098 ),
2099 }),
2100 Effect.as(true),
2101 // Dispatch success owns the replay key even if later bookkeeping fails.
2102 Effect.catchCause((cause) =>
2103 Effect.gen(function* () {
2104 const started = yield* Ref.get(dispatched);
2105 yield* Effect.logWarning("Webhook dispatch failed", { taskId: task.id, cause });
2106 yield* runOutcome(started ? "started" : "failed");
2107 yield* markDeliveryFailed(
2108 deliveryId,
2109 started
2110 ? "The run started, but completion could not be recorded."
2111 : "The run failed to start.",
2112 );
2113 if (!started) {
2114 yield* sql
2115 .withTransaction(
2116 Effect.gen(function* () {
2117 const counted =
2118 yield* sql`UPDATE scheduled_task_webhook_pending SET state = 'failed'
2119 WHERE delivery_id = ${deliveryId} AND state = 'running' RETURNING delivery_id`;
2120 if (counted.length > 0)
2121 yield* sql`UPDATE scheduled_tasks SET run_count = run_count + 1
2122 WHERE task_id = ${task.id} AND created_at = ${task.createdAt}`;
2123 }),
2124 )
2125 .pipe(Effect.mapError(dedupeError));
2126 yield* releaseFailedDelivery(deliveryId);
2127 }
2128 // Pending windows retry the same command id to finish bookkeeping.
2129 return !started;
2130 }),
2131 ),
2132 );
2133 });
2134
2135 // Delivery attributes belong to the enclosing triggerWebhook span.
2136 const observeDelivery = Effect.fnUntraced(function* (
2137 request: WebhookTriggerRequest,
2138 outcome: WebhookDeliveryOutcome,
2139 receivedAt: DateTime.DateTime | undefined,
2140 now: DateTime.DateTime | undefined,
2141 ) {
2142 const source = request.relayDeliveryId === undefined ? "direct" : "relay";
2143 yield* Effect.annotateCurrentSpan({
2144 "scheduled_task.webhook.outcome": outcome,
2145 "scheduled_task.webhook.source": source,
2146 });
2147 yield* Metrics.increment(Metrics.webhookDeliveriesTotal, { outcome, source });
2148 if (request.receivedAt !== undefined && receivedAt !== undefined && now !== undefined) {
2149 const heldMs = Math.max(
2150 0,
2151 DateTime.toEpochMillis(now) - DateTime.toEpochMillis(receivedAt),
2152 );
2153 yield* Effect.annotateCurrentSpan({ "scheduled_task.webhook.held_ms": heldMs });
2154 yield* Metric.update(Metrics.webhookHeldDelay, Duration.millis(heldMs));
2155 }
2156 });
2157
2158 const triggerWebhook: ScheduledTaskService["Service"]["triggerWebhook"] = (request) =>
2159 triggerWebhookUnobserved(request).pipe(
2160 // Accepted deliveries may have queued a coalescing window.
2161 Effect.ensuring(pendingWebhooks.invalidate),
2162 Effect.tapError(() => observeDelivery(request, "error", undefined, undefined)),
2163 Metrics.withMetrics({ timer: Metrics.webhookDeliveryDuration }),
2164 Effect.withSpan("ScheduledTaskService.triggerWebhook", {
2165 attributes: {
2166 "scheduled_task.webhook.method": request.method,
2167 "scheduled_task.webhook.body_bytes": request.body.byteLength,
2168 "scheduled_task.webhook.source":
2169 request.relayDeliveryId === undefined ? "direct" : "relay",
2170 },
2171 }),
2172 );
2173
2174 // triggerWebhook owns the operation span and its task-id annotation.
2175 const triggerWebhookUnobserved = Effect.fnUntraced(function* (request: WebhookTriggerRequest) {
2176 // Refused before any lookup or log, so a blocked hook neither runs nor records a delivery.
2177 if (webhookPolicyBlock !== undefined) {
2178 return yield* observeDelivery(request, "blocked_by_policy", undefined, undefined).pipe(
2179 Effect.as({ _tag: "blocked_by_policy" as const }),
2180 );
2181 }
2182 const taskId = decodeTaskId(request.hookId);
2183 const notFound = observeDelivery(request, "not_found", undefined, undefined).pipe(
2184 Effect.as({ _tag: "not_found" as const }),
2185 );
2186 if (Option.isNone(taskId)) return yield* notFound;
2187 const rows = yield* getRows(taskId.value).pipe(
2188 Effect.mapError((cause) =>
2189 taskError("Could not load schedule task.", { taskId: taskId.value, cause }),
2190 ),
2191 );
2192 const row = rows[0];
2193 // A wrong token is indistinguishable from an unknown hook, so the URL
2194 // does not reveal which hooks exist.
2195 if (
2196 row === undefined ||
2197 row.webhook_token === null ||
2198 !(yield* constantTimeEquals(request.token, row.webhook_token).pipe(
2199 Effect.provideService(Digest.Digest, digest),
2200 ))
2201 ) {
2202 return yield* notFound;
2203 }
2204 const task = yield* decodeRow(row);
2205 yield* Effect.annotateCurrentSpan({ "scheduled_task.id": task.id });
2206 const schedule = task.schedule;
2207 if (schedule.type !== "webhook") return yield* notFound;
2208
2209 const now = yield* localNow;
2210 // Anyone can reach the tunnel directly and set the relay's header, so
2211 // a receive time is never later than now: a future one would pin the
2212 // delivery in the log and slip past the task's max age.
2213 const receivedAt =
2214 request.receivedAt === undefined
2215 ? now
2216 : DateTime.min(
2217 Option.getOrElse(DateTime.make(request.receivedAt), () => now),
2218 now,
2219 );
2220 const observe = (outcome: WebhookDeliveryOutcome) =>
2221 observeDelivery(request, outcome, receivedAt, now);
2222 const deliveryId = ScheduledTaskWebhookDeliveryId.make(
2223 request.relayDeliveryId === undefined
2224 ? `delivery:${yield* random
2225 .uuidV4()
2226 .pipe(
2227 Effect.mapError((cause) => taskError("Could not generate delivery id.", { cause })),
2228 )}`
2229 : `delivery:relay:${request.relayDeliveryId}`,
2230 );
2231 // Claimed in its own table, kept longer than the relay holds a request,
2232 // because the delivery log is trimmed. A rate-limited delivery stays
2233 // held on the relay, so it must give up its claim or the next pass
2234 // would treat it as already delivered.
2235 let claimAcquired = false;
2236 const releaseClaim = <A>(result: A) =>
2237 !claimAcquired || request.relayDeliveryId === undefined
2238 ? Effect.succeed(result)
2239 : sql`
2240 DELETE FROM scheduled_task_webhook_relay_deliveries
2241 WHERE relay_delivery_id = ${request.relayDeliveryId}
2242 `.pipe(Effect.ignore, Effect.as(result));
2243 // From the claim until the run is forked nothing may interrupt: a
2244 // request dropped in between (the relay hangs up after its timeout)
2245 // would leave a claimed delivery that never runs, or a queue slot that
2246 // is never released.
2247 return yield* Effect.uninterruptible(
2248 Effect.gen(function* () {
2249 if (request.relayDeliveryId !== undefined) {
2250 const claimed = yield* sql<{ relay_delivery_id: string }>`
2251 INSERT INTO scheduled_task_webhook_relay_deliveries
2252 (relay_delivery_id, task_id, seen_at)
2253 VALUES (${request.relayDeliveryId}, ${task.id}, ${iso(now)})
2254 ON CONFLICT (relay_delivery_id) DO NOTHING
2255 RETURNING relay_delivery_id
2256 `.pipe(
2257 Effect.mapError((cause) =>
2258 taskError("Could not record webhook delivery.", { taskId: task.id, cause }),
2259 ),
2260 );
2261 if (claimed.length === 0) {
2262 yield* observe("duplicate");
2263 return { _tag: "accepted" as const, deliveryId, outcome: "duplicate" as const };
2264 }
2265 claimAcquired = true;
2266 yield* sql`
2267 DELETE FROM scheduled_task_webhook_relay_deliveries
2268 WHERE seen_at < ${iso(DateTime.subtract(now, { hours: 48 }))}
2269 `.pipe(Effect.ignore);
2270 }
2271 const log = (
2272 outcome: ScheduledTaskWebhookDeliveryOutcome,
2273 details: {
2274 readonly signatureVerified?: boolean;
2275 readonly missing?: ReadonlyArray<string>;
2276 readonly renderedPrompt?: string;
2277 readonly error?: string;
2278 } = {},
2279 ) =>
2280 recordDelivery({
2281 id: deliveryId,
2282 taskId: task.id,
2283 receivedAt: iso(receivedAt),
2284 request,
2285 outcome,
2286 signatureVerified: details.signatureVerified ?? false,
2287 missing: details.missing ?? [],
2288 renderedPrompt: details.renderedPrompt ?? null,
2289 ...(details.error === undefined ? {} : { error: details.error }),
2290 });
2291
2292 // Reject floods before signature work or rejection-log writes; valid
2293 // deliveries give this reservation back once validation succeeds.
2294 const rejectionReservation = { at: DateTime.toEpochMillis(now) };
2295 const rejectionSlot = yield* takeRateSlot(
2296 task.id,
2297 DateTime.toEpochMillis(now),
2298 webhookRejectedRateWindows,
2299 rejectionReservation,
2300 );
2301 if (rejectionSlot !== "allowed") {
2302 if (rejectionSlot === "first_rejected") yield* log("rate_limited");
2303 yield* observe("rate_limited");
2304 return yield* releaseClaim({ _tag: "rate_limited", outcome: "rate_limited" } as const);
2305 }
2306 if (!task.enabled) {
2307 yield* log("disabled");
2308 yield* observe("disabled");
2309 return { _tag: "disabled" as const };
2310 }
2311 const signature = schedule.signature;
2312 if (signature !== null) {
2313 const verified =
2314 row.webhook_secret !== null &&
2315 (yield* verifyWebhookSignature({
2316 signature,
2317 secret: row.webhook_secret,
2318 headers: request.headers,
2319 body: request.body,
2320 }).pipe(Effect.provideService(Mac.Mac, mac)));
2321 if (!verified) {
2322 yield* log("rejected_signature");
2323 yield* observe("rejected_signature");
2324 return { _tag: "rejected_signature" as const };
2325 }
2326 }
2327 const maxAgeMinutes = schedule.maxDeliveryAgeMinutes ?? null;
2328 if (
2329 maxAgeMinutes !== null &&
2330 DateTime.toEpochMillis(now) - DateTime.toEpochMillis(receivedAt) >
2331 maxAgeMinutes * 60_000
2332 ) {
2333 yield* log("expired", { signatureVerified: signature !== null });
2334 yield* observe("expired");
2335 return { _tag: "expired" as const };
2336 }
2337
2338 const refundRejectionSlot = Ref.update(webhookRejectedRateWindows, (windows) => {
2339 const current = windows.get(task.id);
2340 if (current === undefined) return windows;
2341 return new Map(windows).set(task.id, {
2342 ...current,
2343 accepted: current.accepted.filter((entry) => entry !== rejectionReservation),
2344 });
2345 });
2346
2347 const failedFilter = findFailedWebhookFilter(schedule.filters ?? [], request);
2348 if (failedFilter !== null) {
2349 if (yield* takeFilteredLogSlot(task.id, DateTime.toEpochMillis(now))) {
2350 yield* log("filtered", {
2351 signatureVerified: signature !== null,
2352 error: describeFailedWebhookFilter(failedFilter),
2353 });
2354 }
2355 yield* refundRejectionSlot;
2356 yield* observe("filtered");
2357 return yield* releaseClaim({
2358 _tag: "accepted" as const,
2359 deliveryId,
2360 outcome: "filtered" as const,
2361 });
2362 }
2363
2364 const runWindow = (yield* Ref.get(webhookRateWindows)).get(task.id);
2365 if (
2366 schedule.dedupe === undefined &&
2367 (runWindow?.accepted.filter((entry) => DateTime.toEpochMillis(now) - entry.at < 60_000)
2368 .length ?? 0) >= WEBHOOK_RATE_LIMIT_PER_MINUTE
2369 ) {
2370 if ((yield* takeRateSlot(task.id, DateTime.toEpochMillis(now))) === "first_rejected")
2371 yield* log("rate_limited");
2372 yield* observe("rate_limited");
2373 return yield* releaseClaim({ _tag: "rate_limited", outcome: "rate_limited" } as const);
2374 }
2375 const reserve = takeRateSlot(task.id, DateTime.toEpochMillis(now)).pipe(
2376 Effect.flatMap((slot) => (slot === "allowed" ? Effect.void : Effect.fail(slot))),
2377 );
2378 const queueKey = `${task.id}\u0000${task.createdAt}`;
2379 const checkCapacity = Effect.gen(function* () {
2380 const [row] = yield* sql<{
2381 count: number;
2382 }>`SELECT COUNT(*) AS count FROM scheduled_task_webhook_pending
2383 WHERE task_id = ${task.id} AND state IN ('pending', 'dispatching', 'running')`;
2384 if (
2385 (row?.count ?? 0) + ((yield* Ref.get(webhookQueued)).get(queueKey) ?? 0) >=
2386 WEBHOOK_MAX_QUEUED_PER_TASK
2387 )
2388 return yield* Effect.fail("queue_full" as const);
2389 }).pipe(
2390 Effect.mapError((cause) => (cause === "queue_full" ? cause : dedupeError(cause))),
2391 );
2392 const admit = <A, E>(effect: Effect.Effect<A, E>) =>
2393 sql.withTransaction(effect).pipe(
2394 Effect.catch((cause) =>
2395 cause === "first_rejected" || cause === "rejected" || cause === "queue_full"
2396 ? Effect.succeed({
2397 outcome:
2398 cause === "queue_full"
2399 ? ("queue_full" as const)
2400 : ("rate_limited" as const),
2401 log: cause === "first_rejected",
2402 })
2403 : Effect.fail(isScheduledTaskError(cause) ? cause : dedupeError(cause)),
2404 ),
2405 );
2406 const deduped =
2407 schedule.dedupe === undefined
2408 ? "run"
2409 : yield* admit(
2410 Effect.gen(function* () {
2411 const outcome = yield* acceptWebhookDedupe(
2412 task,
2413 request,
2414 deliveryId,
2415 DateTime.toEpochMillis(now),
2416 reserve,
2417 takeDedupeLogSlot(task.id, DateTime.toEpochMillis(now)),
2418 checkCapacity,
2419 (ownerId) => replaceDeliveryRequest(ownerId, request),
2420 ).pipe(Effect.provideService(SqlClient.SqlClient, sql));
2421 if (
2422 outcome === "pending" ||
2423 outcome === "coalesced" ||
2424 (outcome === "duplicate" &&
2425 (yield* takeDedupeLogSlot(task.id, DateTime.toEpochMillis(now))))
2426 )
2427 yield* log(outcome === "pending" ? "accepted" : outcome, {
2428 signatureVerified: signature !== null,
2429 });
2430 return outcome;
2431 }),
2432 );
2433 if (typeof deduped !== "string") {
2434 yield* refundRejectionSlot;
2435 if (deduped.log) yield* log("rate_limited");
2436 yield* observe(deduped.outcome);
2437 return yield* releaseClaim({ _tag: "rate_limited", outcome: deduped.outcome } as const);
2438 }
2439 if (deduped !== "run") {
2440 if (deduped !== "coalesced" && deduped !== "coalesced_unlogged")
2441 yield* refundRejectionSlot;
2442 const outcome =
2443 deduped === "pending"
2444 ? "accepted"
2445 : deduped === "coalesced_unlogged"
2446 ? "coalesced"
2447 : deduped;
2448 yield* observe(outcome);
2449 if (deduped === "duplicate")
2450 return yield* releaseClaim({ _tag: "accepted", deliveryId, outcome } as const);
2451 claimAcquired = false;
2452 return { _tag: "accepted", deliveryId, outcome } as const;
2453 }
2454 const frameId = yield* random.randomBytes(6).pipe(
2455 Effect.map((bytes) => Buffer.from(bytes).toString("hex")),
2456 Effect.mapError((cause) => taskError("Could not generate frame id.", { cause })),
2457 );
2458 const rendered = renderWebhookPrompt(task.prompt, request, frameId);
2459 // Providers trim the prompt before checking, so whitespace around
2460 // it is free.
2461 if (rendered.prompt.trim().length > PROVIDER_SEND_TURN_MAX_INPUT_CHARS) {
2462 yield* log("dispatch_failed", {
2463 signatureVerified: signature !== null,
2464 missing: rendered.missing,
2465 renderedPrompt: rendered.prompt,
2466 error: PROMPT_TOO_LONG,
2467 });
2468 yield* releaseReplay(deliveryId);
2469 yield* observe("prompt_too_long");
2470 return yield* releaseClaim({
2471 _tag: "accepted" as const,
2472 deliveryId,
2473 outcome: "prompt_too_long" as const,
2474 });
2475 }
2476 // Bound the deliveries one task holds, so steady traffic to a stuck
2477 // task cannot pile up parked fibers.
2478 yield* refundRejectionSlot;
2479 let queueReserved = false;
2480 const release = Effect.suspend(() => {
2481 if (!queueReserved) return Effect.void;
2482 queueReserved = false;
2483 return Ref.update(webhookQueued, (counts) => {
2484 const next = new Map(counts);
2485 const count = (next.get(queueKey) ?? 1) - 1;
2486 if (count <= 0) next.delete(queueKey);
2487 else next.set(queueKey, count);
2488 return next;
2489 });
2490 });
2491 // Serialize immediate reservations with persisted window admission.
2492 const reserveImmediate = Effect.gen(function* () {
2493 yield* checkCapacity;
2494 yield* reserve;
2495 yield* Ref.update(webhookQueued, (counts) =>
2496 new Map(counts).set(queueKey, (counts.get(queueKey) ?? 0) + 1),
2497 );
2498 queueReserved = true;
2499 });
2500 const admitted = yield* admit(
2501 Effect.gen(function* () {
2502 const outcome = yield* acceptImmediateWebhookDedupe(
2503 task,
2504 request,
2505 deliveryId,
2506 DateTime.toEpochMillis(now),
2507 reserveImmediate,
2508 ).pipe(Effect.provideService(SqlClient.SqlClient, sql));
2509 if (outcome === "duplicate") {
2510 if (yield* takeDedupeLogSlot(task.id, DateTime.toEpochMillis(now)))
2511 yield* log("duplicate", { signatureVerified: signature !== null });
2512 } else
2513 yield* log("accepted", {
2514 signatureVerified: signature !== null,
2515 missing: rendered.missing,
2516 renderedPrompt: rendered.prompt,
2517 });
2518 return outcome;
2519 }),
2520 ).pipe(Effect.onError(() => release));
2521 if (typeof admitted !== "string") {
2522 yield* release;
2523 if (admitted.log) yield* log("rate_limited");
2524 yield* observe(admitted.outcome);
2525 return yield* releaseClaim({
2526 _tag: "rate_limited",
2527 outcome: admitted.outcome,
2528 } as const);
2529 }
2530 if (admitted === "duplicate") {
2531 yield* release;
2532 yield* observe("duplicate");
2533 return yield* releaseClaim({
2534 _tag: "accepted",
2535 deliveryId,
2536 outcome: "duplicate",
2537 } as const);
2538 }
2539 yield* observe("accepted");
2540 const permit = yield* webhookPermit(task.id);
2541 yield* runWebhookDelivery(task, deliveryId, rendered.prompt).pipe(
2542 permit.withPermits(1),
2543 Effect.ensuring(release),
2544 Effect.forkIn(serviceScope),
2545 );
2546 // Dispatch owns the accepted delivery now; retries must keep its claim.
2547 claimAcquired = false;
2548 return { _tag: "accepted" as const, deliveryId, outcome: "accepted" as const };
2549 }).pipe(
2550 Effect.onError(() =>
2551 Effect.all([
2552 releaseClaim(undefined),
2553 (request.relayDeliveryId === undefined || claimAcquired
2554 ? releaseReplay(deliveryId)
2555 : Effect.void
2556 ).pipe(Effect.ignore),
2557 ]),
2558 ),
2559 ),
2560 );
2561 });
2562
2563 const dispatchPending = Effect.fn("ScheduledTaskService.dispatchPendingWebhooks")(function* () {
2564 const now = DateTime.toEpochMillis(yield* localNow);
2565 if (yield* pendingWebhooks.isIdle(now)) return;
2566 const readVersion = yield* pendingWebhooks.version;
2567 yield* sql`UPDATE scheduled_task_webhook_pending SET state = 'dispatching' WHERE state = 'pending' AND due_at <= ${now}`;
2568 const tasks = yield* sql<{ task_id: ScheduledTaskId }>`
2569 SELECT DISTINCT task_id FROM scheduled_task_webhook_pending WHERE state IN ('dispatching', 'running', 'completed', 'failed')
2570 `.pipe(NodeSqliteClient.readOnly);
2571 // Workers can leave rows behind to retry, so keep polling while any are claimed.
2572 if (tasks.length === 0) {
2573 const [next] = yield* sql<{ due_at: number | null }>`
2574 SELECT MIN(due_at) AS due_at FROM scheduled_task_webhook_pending WHERE state = 'pending'
2575 `.pipe(NodeSqliteClient.readOnly);
2576 yield* pendingWebhooks.record(readVersion, next?.due_at ?? Infinity, now);
2577 }
2578 yield* Effect.forEach(
2579 tasks,
2580 ({ task_id: taskId }) => {
2581 // One worker per task waits on its permit, then drains persisted rows in order.
2582 // Other tasks remain independent and ticks never reload an active task's payloads.
2583 const dispatch = Effect.gen(function* () {
2584 while (true) {
2585 const [entry] = yield* sql<{
2586 delivery_id: string;
2587 task_json: string;
2588 request_json: string;
2589 state: string;
2590 }>`
2591 SELECT delivery_id, task_json, request_json, state FROM scheduled_task_webhook_pending
2592 WHERE task_id = ${taskId} AND state IN ('dispatching', 'running', 'completed', 'failed') ORDER BY due_at, rowid LIMIT 1
2593 `.pipe(NodeSqliteClient.readOnly);
2594 if (entry === undefined) return;
2595 const remove = sql`DELETE FROM scheduled_task_webhook_pending WHERE delivery_id = ${entry.delivery_id}`;
2596 if (entry.state === "completed" || entry.state === "failed") {
2597 if (entry.state === "failed") {
2598 yield* markDeliveryFailed(entry.delivery_id, "The run failed to start.");
2599 yield* releaseFailedDelivery(entry.delivery_id);
2600 }
2601 yield* remove;
2602 continue;
2603 }
2604 const prepared = yield* Effect.gen(function* () {
2605 // A running row was already sent, so recovery keeps the prompt it was sent with.
2606 const current = entry.state === "dispatching" ? yield* findTask(taskId) : null;
2607 const snapshot = yield* decodePendingTask(entry.task_json);
2608 // The snapshot keeps the identity runTask checks; the prompt is the one saved now.
2609 const task =
2610 current?.createdAt === snapshot.createdAt && current.prompt !== snapshot.prompt
2611 ? { ...snapshot, prompt: current.prompt }
2612 : snapshot;
2613 const request = yield* decodePendingRequest(entry.request_json);
2614 const frameId = Buffer.from(yield* random.randomBytes(6)).toString("hex");
2615 return {
2616 task,
2617 taskJson: task === snapshot ? null : yield* encodePendingTask(task),
2618 rendered: renderWebhookPrompt(task.prompt, request, frameId),
2619 };
2620 }).pipe(Effect.exit);
2621 if (Exit.isFailure(prepared)) {
2622 if (Cause.hasInterruptsOnly(prepared.cause))
2623 return yield* Effect.failCause(prepared.cause);
2624 yield* markDeliveryFailed(
2625 entry.delivery_id,
2626 "The pending webhook payload could not be prepared.",
2627 );
2628 yield* releaseFailedDelivery(entry.delivery_id);
2629 yield* remove;
2630 continue;
2631 }
2632 const { task, taskJson, rendered } = prepared.value;
2633 if (rendered.prompt.trim().length > PROVIDER_SEND_TURN_MAX_INPUT_CHARS) {
2634 yield* markDeliveryFailed(entry.delivery_id, PROMPT_TOO_LONG);
2635 yield* releaseFailedDelivery(entry.delivery_id);
2636 } else {
2637 if (taskJson !== null)
2638 yield* sql`UPDATE scheduled_task_webhook_pending SET task_json = ${taskJson} WHERE delivery_id = ${entry.delivery_id}`;
2639 yield* sql`UPDATE scheduled_task_webhook_deliveries SET outcome = 'accepted', error = NULL, rendered_prompt = ${rendered.prompt.slice(0, WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT)}, missing_fields_json = ${encodeMissingFieldsJson(rendered.missing)} WHERE delivery_id = ${entry.delivery_id}`;
2640 if (!(yield* runWebhookDelivery(task, entry.delivery_id, rendered.prompt))) return;
2641 }
2642 yield* remove;
2643 }
2644 });
2645 // The worker owns its reservation, so an interrupt anywhere still releases it.
2646 return Effect.acquireUseRelease(
2647 Ref.modify(pendingInFlight, (active) =>
2648 active.has(taskId)
2649 ? ([false, active] as const)
2650 : ([true, new Set(active).add(taskId)] as const),
2651 ),
2652 (reserved) =>
2653 reserved
2654 ? webhookPermit(taskId).pipe(
2655 Effect.flatMap((permit) => dispatch.pipe(permit.withPermits(1))),
2656 Effect.catch((cause) =>
2657 Effect.logWarning("Pending webhook dispatch failed", { cause }),
2658 ),
2659 Effect.withSpan("ScheduledTaskService.dispatchPendingWebhook", {
2660 root: true,
2661 }),
2662 )
2663 : Effect.void,
2664 (reserved) =>
2665 reserved
2666 ? Ref.update(pendingInFlight, (active) => {
2667 const next = new Set(active);
2668 next.delete(taskId);
2669 return next;
2670 })
2671 : Effect.void,
2672 ).pipe(Effect.forkIn(serviceScope));
2673 },
2674 { discard: true },
2675 );
2676 });
2677 yield* scheduler.register("webhook-coalescing", dispatchPending());
2678
2679 return ScheduledTaskService.of({
2680 list,
2681 subscribeList,
2682 upsert,
2683 setEnabled,
2684 delete: deleteTask,
2685 deleteForThread,
2686 moveThreadTasks,
2687 runNow,
2688 rotateWebhookToken,
2689 listWebhookDeliveries,
2690 getWebhookDelivery,
2691 triggerWebhook,
2692 });
2693 }),
2694 );