The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 04:45 UTC
| 1 | import { |
| 2 | CommandId, |
| 3 | MessageId, |
| 4 | PROVIDER_SEND_TURN_MAX_INPUT_CHARS, |
| 5 | type ProjectId, |
| 6 | ScheduledTask, |
| 7 | ScheduledTaskError, |
| 8 | ScheduledTaskId, |
| 9 | ThreadId, |
| 10 | ScheduledTaskWebhookDeliveryId, |
| 11 | type ScheduledTaskDeleteInput, |
| 12 | type ScheduledTaskDeleteResult, |
| 13 | type ScheduledTaskGetWebhookDeliveryInput, |
| 14 | type ScheduledTaskGetWebhookDeliveryResult, |
| 15 | type ScheduledTaskListWebhookDeliveriesInput, |
| 16 | type ScheduledTaskListWebhookDeliveriesResult, |
| 17 | type ScheduledTaskRotateWebhookTokenInput, |
| 18 | type ScheduledTaskWebhookDeliveryOutcome, |
| 19 | type ScheduledTaskWebhookFilter, |
| 20 | type ScheduledTaskListResult, |
| 21 | type ScheduledTaskMutationResult, |
| 22 | type ScheduledTaskRunNowInput, |
| 23 | type ScheduledTaskRunNowResult, |
| 24 | type ScheduledTaskSetEnabledInput, |
| 25 | type ScheduledTaskThreadMode, |
| 26 | type ScheduledTaskUpsertInput, |
| 27 | managedPolicyDeniedMessage, |
| 28 | scheduledTaskThreadMode, |
| 29 | managedRuntimeModeDenial, |
| 30 | } from "@t3tools/contracts"; |
| 31 | import * as Cause from "effect/Cause"; |
| 32 | import * as Context from "effect/Context"; |
| 33 | import * as Digest from "effect/crypto/Digest"; |
| 34 | import * as Mac from "effect/crypto/Mac"; |
| 35 | import * as SecureRandom from "effect/crypto/SecureRandom"; |
| 36 | import * as DateTime from "effect/DateTime"; |
| 37 | import * as Effect from "effect/Effect"; |
| 38 | import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; |
| 39 | import * as Exit from "effect/Exit"; |
| 40 | import * as Duration from "effect/Duration"; |
| 41 | import * as Metric from "effect/Metric"; |
| 42 | import * as Layer from "effect/Layer"; |
| 43 | import * as Option from "effect/Option"; |
| 44 | import * as PubSub from "effect/PubSub"; |
| 45 | import * as Ref from "effect/Ref"; |
| 46 | import * as Result from "effect/Result"; |
| 47 | import * as Schema from "effect/Schema"; |
| 48 | import * as Semaphore from "effect/Semaphore"; |
| 49 | import * as Stream from "effect/Stream"; |
| 50 | import * as SqlClient from "effect/sql/SqlClient"; |
| 51 | |
| 52 | import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts"; |
| 53 | import * as ManagedProjectFolders from "../project/ManagedProjectFolders.ts"; |
| 54 | import * as Metrics from "../observability/Metrics.ts"; |
| 55 | import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts"; |
| 56 | import { OrchestratorDispatchError } from "../orchestration-v2/Orchestrator.ts"; |
| 57 | import { userFacingDispatchErrorMessage } from "../orchestration-v2/UserFacingErrors.ts"; |
| 58 | import * as SecretRequests from "../secrets/SecretRequests.ts"; |
| 59 | import * as ManagedPolicy from "../policy/ManagedPolicy.ts"; |
| 60 | import * as DueHint from "../scheduling/DueHint.ts"; |
| 61 | import * as Scheduler from "../scheduling/Scheduler.ts"; |
| 62 | import { latestRootProviderFailure } from "@t3tools/shared/orchestrationV2ThreadError"; |
| 63 | import { isMissedFixedTimeRun, isSameSchedule, nextScheduledRunAt } from "./Schedule.ts"; |
| 64 | import { |
| 65 | redactHeaders, |
| 66 | redactQuery, |
| 67 | renderWebhookPrompt, |
| 68 | type WebhookRequest, |
| 69 | } from "./webhookTemplate.ts"; |
| 70 | import { |
| 71 | acceptWebhookDedupe, |
| 72 | acceptImmediateWebhookDedupe, |
| 73 | releaseWebhookReplay, |
| 74 | decodePendingTask, |
| 75 | encodePendingTask, |
| 76 | decodePendingRequest, |
| 77 | dedupeError, |
| 78 | } from "./webhookDedupe.ts"; |
| 79 | import { describeFailedWebhookFilter, findFailedWebhookFilter } from "./webhookFilter.ts"; |
| 80 | import { constantTimeEquals, verifyWebhookSignature } from "./webhookVerification.ts"; |
| 81 | |
| 82 | export const WEBHOOK_ROUTE_PREFIX = "/api/hooks"; |
| 83 | const isScheduledTaskError = Schema.is(ScheduledTaskError); |
| 84 | const isOrchestratorDispatchError = Schema.is(OrchestratorDispatchError); |
| 85 | const PROMPT_TOO_LONG = "The filled-in prompt is too long."; |
| 86 | const WEBHOOK_DELIVERY_RETENTION = 50; |
| 87 | const WEBHOOK_DEDUPE_DELIVERY_RETENTION = 20; |
| 88 | const WEBHOOK_FILTERED_DELIVERY_RETENTION = 20; |
| 89 | const WEBHOOK_DEDUPE_LOG_PER_MINUTE = 10; |
| 90 | const WEBHOOK_FILTERED_LOG_PER_MINUTE = 10; |
| 91 | const WEBHOOK_DELIVERY_LOG_BODY_LIMIT = 64 * 1024; |
| 92 | const WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT = 64 * 1024; |
| 93 | const WEBHOOK_MAX_QUEUED_PER_TASK = 20; |
| 94 | // Enforce the relay's per-task/minute limit here too; tunnel hostnames are public. |
| 95 | const WEBHOOK_RATE_LIMIT_PER_MINUTE = 60; |
| 96 | const LEGACY_DELIVERY_OUTCOMES: ReadonlySet<string> = new Set([ |
| 97 | "accepted", |
| 98 | "dispatch_failed", |
| 99 | "rejected_signature", |
| 100 | "disabled", |
| 101 | "rate_limited", |
| 102 | "expired", |
| 103 | ]); |
| 104 | |
| 105 | interface WebhookOrigin { |
| 106 | readonly relayHookBaseUrl: string | null; |
| 107 | } |
| 108 | |
| 109 | const ENDPOINT_KEY = /^[0-9a-f]{16}$/; |
| 110 | |
| 111 | export function relayHookBaseUrl(input: { |
| 112 | readonly relayUrl: string; |
| 113 | readonly tunnelName: string | undefined; |
| 114 | }): string | null { |
| 115 | const endpointKey = input.tunnelName?.split("-").at(-1); |
| 116 | const relayUrl = input.relayUrl.replace(/\/+$/, ""); |
| 117 | if (relayUrl === "" || endpointKey === undefined || !ENDPOINT_KEY.test(endpointKey)) { |
| 118 | return null; |
| 119 | } |
| 120 | return `${relayUrl}/v1/hooks/${endpointKey}`; |
| 121 | } |
| 122 | |
| 123 | export class ScheduledTaskWebhookOrigin extends Context.Reference<Effect.Effect<WebhookOrigin>>( |
| 124 | "t3/scheduledTasks/ScheduledTaskWebhookOrigin", |
| 125 | { |
| 126 | defaultValue: () => Effect.succeed({ relayHookBaseUrl: null }), |
| 127 | }, |
| 128 | ) {} |
| 129 | |
| 130 | class WebhookDeliverySkipped extends Schema.TaggedError<WebhookDeliverySkipped>()( |
| 131 | "WebhookDeliverySkipped", |
| 132 | { reason: Schema.String }, |
| 133 | ) {} |
| 134 | |
| 135 | interface RateWindow { |
| 136 | readonly accepted: ReadonlyArray<{ readonly at: number }>; |
| 137 | readonly rejectedLogged: boolean; |
| 138 | } |
| 139 | |
| 140 | export interface WebhookTriggerRequest extends WebhookRequest { |
| 141 | readonly hookId: string; |
| 142 | readonly token: string; |
| 143 | readonly body: Uint8Array; |
| 144 | readonly relayDeliveryId?: string; |
| 145 | readonly receivedAt?: string; |
| 146 | } |
| 147 | |
| 148 | export type WebhookDeliveryOutcome = |
| 149 | | "accepted" |
| 150 | | "duplicate" |
| 151 | | "coalesced" |
| 152 | | "prompt_too_long" |
| 153 | | "queue_full" |
| 154 | | "rate_limited" |
| 155 | | "disabled" |
| 156 | | "rejected_signature" |
| 157 | | "expired" |
| 158 | | "filtered" |
| 159 | | "not_found" |
| 160 | | "blocked_by_policy" |
| 161 | | "error"; |
| 162 | |
| 163 | export type WebhookTriggerResult = |
| 164 | | { |
| 165 | readonly _tag: "accepted"; |
| 166 | readonly deliveryId: ScheduledTaskWebhookDeliveryId; |
| 167 | readonly outcome: "accepted" | "duplicate" | "coalesced" | "prompt_too_long" | "filtered"; |
| 168 | } |
| 169 | | { readonly _tag: "not_found" } |
| 170 | | { readonly _tag: "rejected_signature" } |
| 171 | | { readonly _tag: "disabled" } |
| 172 | | { readonly _tag: "blocked_by_policy" } |
| 173 | | { |
| 174 | readonly _tag: "rate_limited"; |
| 175 | readonly outcome: "rate_limited" | "queue_full"; |
| 176 | } |
| 177 | | { readonly _tag: "expired" }; |
| 178 | |
| 179 | const webhookFilters = (filters: ReadonlyArray<ScheduledTaskWebhookFilter> | undefined) => |
| 180 | filters?.length ? { filters } : {}; |
| 181 | |
| 182 | const decodeTask = Schema.decodeUnknownEffect(ScheduledTask); |
| 183 | const decodeTaskId = Schema.decodeUnknownOption(ScheduledTaskId); |
| 184 | const decodeScheduleJson = Schema.decodeUnknownEffect( |
| 185 | Schema.fromJsonString(ScheduledTask.fields.schedule), |
| 186 | ); |
| 187 | const decodeWorkspaceStrategyJson = Schema.decodeUnknownEffect( |
| 188 | Schema.fromJsonString(ScheduledTask.fields.workspaceStrategy), |
| 189 | ); |
| 190 | const decodeModelSelectionJson = Schema.decodeUnknownEffect( |
| 191 | Schema.fromJsonString(ScheduledTask.fields.modelSelection), |
| 192 | ); |
| 193 | const HeadersJson = Schema.fromJsonString(Schema.Record(Schema.String, Schema.String)); |
| 194 | const MissingFieldsJson = Schema.fromJsonString(Schema.Array(Schema.String)); |
| 195 | const decodeHeadersJson = Schema.decodeUnknownOption(HeadersJson); |
| 196 | const decodeMissingFieldsJson = Schema.decodeUnknownOption(MissingFieldsJson); |
| 197 | const encodeHeadersJson = Schema.encodeSync(HeadersJson); |
| 198 | const encodeMissingFieldsJson = Schema.encodeSync(MissingFieldsJson); |
| 199 | |
| 200 | /** |
| 201 | * Clients that predate thread modes send none, and may send a thread loaded before the first run |
| 202 | * bound one. An edit keeps the stored mode; binding a thread to a new-thread task continues it. |
| 203 | */ |
| 204 | function resolveThreadMode( |
| 205 | input: Pick<ScheduledTaskUpsertInput, "threadId" | "threadMode">, |
| 206 | existing: Pick<ScheduledTask, "threadId" | "threadMode"> | null, |
| 207 | ): ScheduledTaskThreadMode { |
| 208 | if (input.threadMode !== undefined) return input.threadMode; |
| 209 | const stored = existing === null ? "new_thread" : scheduledTaskThreadMode(existing); |
| 210 | return stored === "new_thread" && input.threadId != null ? "continue" : stored; |
| 211 | } |
| 212 | |
| 213 | interface ScheduledTaskRow { |
| 214 | readonly task_id: string; |
| 215 | readonly title: string; |
| 216 | readonly prompt: string; |
| 217 | readonly enabled: number; |
| 218 | readonly schedule_json: string; |
| 219 | readonly project_id: string; |
| 220 | readonly thread_id: string | null; |
| 221 | readonly thread_mode: string; |
| 222 | readonly workspace_strategy_json: string; |
| 223 | readonly model_selection_json: string; |
| 224 | readonly runtime_mode: string; |
| 225 | readonly interaction_mode: string; |
| 226 | readonly created_by: string; |
| 227 | readonly creation_source: string; |
| 228 | readonly created_at: string; |
| 229 | readonly updated_at: string; |
| 230 | readonly next_run_at: string | null; |
| 231 | readonly last_run_at: string | null; |
| 232 | readonly last_run_status: string; |
| 233 | readonly last_run_error: string | null; |
| 234 | readonly run_count: number; |
| 235 | readonly webhook_token: string | null; |
| 236 | readonly webhook_secret: string | null; |
| 237 | } |
| 238 | |
| 239 | interface WebhookDeliveryRow { |
| 240 | readonly delivery_id: string; |
| 241 | readonly task_id: string; |
| 242 | readonly received_at: string; |
| 243 | readonly method: string; |
| 244 | readonly query: string; |
| 245 | readonly headers_json: string; |
| 246 | readonly body: string; |
| 247 | readonly body_bytes: number; |
| 248 | readonly body_truncated: number; |
| 249 | readonly outcome: string; |
| 250 | readonly signature_verified: number; |
| 251 | readonly missing_fields_json: string; |
| 252 | readonly rendered_prompt: string | null; |
| 253 | readonly pending_window: number; |
| 254 | readonly error: string | null; |
| 255 | } |
| 256 | |
| 257 | export class ScheduledTaskService extends Context.Service< |
| 258 | ScheduledTaskService, |
| 259 | { |
| 260 | readonly list: () => Effect.Effect<ScheduledTaskListResult, ScheduledTaskError>; |
| 261 | readonly subscribeList: () => Stream.Stream<ScheduledTaskListResult, ScheduledTaskError>; |
| 262 | readonly upsert: ( |
| 263 | input: ScheduledTaskUpsertInput, |
| 264 | ) => Effect.Effect<ScheduledTaskMutationResult, ScheduledTaskError>; |
| 265 | readonly setEnabled: ( |
| 266 | input: ScheduledTaskSetEnabledInput, |
| 267 | ) => Effect.Effect<ScheduledTaskMutationResult, ScheduledTaskError>; |
| 268 | readonly delete: ( |
| 269 | input: ScheduledTaskDeleteInput, |
| 270 | ) => Effect.Effect<ScheduledTaskDeleteResult, ScheduledTaskError>; |
| 271 | /** Deletes only tasks that still target this thread, including their webhook deliveries. */ |
| 272 | readonly deleteForThread: (threadId: ThreadId) => Effect.Effect<void, ScheduledTaskError>; |
| 273 | /** Points tasks bound to these threads at the project the threads moved to. */ |
| 274 | readonly moveThreadTasks: (input: { |
| 275 | readonly threadIds: ReadonlyArray<ThreadId>; |
| 276 | readonly projectId: ProjectId; |
| 277 | }) => Effect.Effect<void, ScheduledTaskError>; |
| 278 | /** The dispatched thread, or null if dispatch failed; leaves the task's binding unchanged. */ |
| 279 | readonly runNow: ( |
| 280 | input: ScheduledTaskRunNowInput, |
| 281 | ) => Effect.Effect< |
| 282 | ScheduledTaskRunNowResult & { readonly threadId: ThreadId | null }, |
| 283 | ScheduledTaskError |
| 284 | >; |
| 285 | readonly rotateWebhookToken: ( |
| 286 | input: ScheduledTaskRotateWebhookTokenInput, |
| 287 | ) => Effect.Effect<ScheduledTaskMutationResult, ScheduledTaskError>; |
| 288 | readonly listWebhookDeliveries: ( |
| 289 | input: ScheduledTaskListWebhookDeliveriesInput, |
| 290 | ) => Effect.Effect<ScheduledTaskListWebhookDeliveriesResult, ScheduledTaskError>; |
| 291 | readonly getWebhookDelivery: ( |
| 292 | input: ScheduledTaskGetWebhookDeliveryInput, |
| 293 | ) => Effect.Effect<ScheduledTaskGetWebhookDeliveryResult, ScheduledTaskError>; |
| 294 | /** Verifies, logs and dispatches; returns after logging while the run continues in the background. */ |
| 295 | readonly triggerWebhook: ( |
| 296 | request: WebhookTriggerRequest, |
| 297 | ) => Effect.Effect<WebhookTriggerResult, ScheduledTaskError>; |
| 298 | } |
| 299 | >()("t3/scheduledTasks/ScheduledTaskService") {} |
| 300 | |
| 301 | function taskError(message: string, input?: { taskId?: ScheduledTaskId; cause?: unknown }) { |
| 302 | return new ScheduledTaskError({ |
| 303 | message, |
| 304 | ...(input?.taskId === undefined ? {} : { taskId: input.taskId }), |
| 305 | ...(input?.cause === undefined ? {} : { cause: input.cause }), |
| 306 | }); |
| 307 | } |
| 308 | |
| 309 | function iso(value: DateTime.DateTime): string { |
| 310 | return DateTime.formatIso(DateTime.toUtc(value)); |
| 311 | } |
| 312 | |
| 313 | const RUN_OUTCOME_WAIT_MS = 24 * 60 * 60 * 1_000; |
| 314 | const HELD_OCCURRENCE_ERROR = |
| 315 | "A previous occurrence is held in the thread queue. Resume or remove it to run this task again."; |
| 316 | const UNSUCCESSFUL_RUN_ERRORS = { |
| 317 | failed: "The run failed.", |
| 318 | interrupted: "The run was stopped.", |
| 319 | cancelled: "The run was cancelled.", |
| 320 | rolled_back: "The run was rolled back.", |
| 321 | } as const; |
| 322 | const localNow = DateTime.withCurrentZoneLocal(DateTime.nowInCurrentZone); |
| 323 | |
| 324 | function nextRunAt( |
| 325 | task: Pick<ScheduledTask, "enabled" | "schedule">, |
| 326 | from: DateTime.DateTime, |
| 327 | ): string | null { |
| 328 | if (!task.enabled) return null; |
| 329 | // Decoded intervals can overflow DateTime; unrepresentable occurrences have no next run. |
| 330 | try { |
| 331 | const next = nextScheduledRunAt(task.schedule, from); |
| 332 | return next !== null && Number.isFinite(DateTime.toEpochMillis(next)) ? iso(next) : null; |
| 333 | } catch { |
| 334 | return null; |
| 335 | } |
| 336 | } |
| 337 | |
| 338 | /** |
| 339 | * Next run after an interrupted run. A one-time task counts from the run's start |
| 340 | * (`lastRunAt`), so a manual run that began before its time does not use it up. |
| 341 | */ |
| 342 | function nextRunAfterRun( |
| 343 | task: Pick<ScheduledTask, "enabled" | "schedule" | "lastRunAt">, |
| 344 | now: DateTime.DateTime, |
| 345 | ): string | null { |
| 346 | const startedAt = |
| 347 | task.schedule.type === "once" && task.lastRunAt !== null |
| 348 | ? DateTime.make(task.lastRunAt) |
| 349 | : Option.none(); |
| 350 | return nextRunAt( |
| 351 | task, |
| 352 | Option.getOrElse(startedAt, () => now), |
| 353 | ); |
| 354 | } |
| 355 | |
| 356 | /** An enabled one-time task with a past time would never run, so saving it is an error. */ |
| 357 | function requireFutureOnce( |
| 358 | task: Pick<ScheduledTask, "enabled" | "schedule">, |
| 359 | now: DateTime.DateTime, |
| 360 | taskId: ScheduledTaskId, |
| 361 | ) { |
| 362 | return task.enabled && task.schedule.type === "once" && nextRunAt(task, now) === null |
| 363 | ? taskError("The run time has passed. Pick a time in the future.", { taskId }) |
| 364 | : Effect.void; |
| 365 | } |
| 366 | |
| 367 | function errorMessage(error: unknown): string { |
| 368 | const failure = Cause.isCause(error) ? Cause.squash(error) : error; |
| 369 | return failure instanceof Error ? failure.message : String(failure); |
| 370 | } |
| 371 | |
| 372 | function dispatchErrorMessage(cause: Cause.Cause<unknown>): string { |
| 373 | const failure = Cause.squash(cause); |
| 374 | // Direct orchestration refusals are user-facing; nested defects can contain private data. |
| 375 | return isOrchestratorDispatchError(failure) && typeof failure.cause === "string" |
| 376 | ? (userFacingDispatchErrorMessage(failure.cause) ?? errorMessage(failure)) |
| 377 | : errorMessage(failure); |
| 378 | } |
| 379 | |
| 380 | function webhookPath(taskId: string, token: string): string { |
| 381 | return `${WEBHOOK_ROUTE_PREFIX}/${encodeURIComponent(taskId)}/${token}`; |
| 382 | } |
| 383 | |
| 384 | function webhookEndpoint( |
| 385 | row: Pick<ScheduledTaskRow, "task_id" | "webhook_token" | "webhook_secret">, |
| 386 | origin: WebhookOrigin | null, |
| 387 | ): ScheduledTask["webhook"] { |
| 388 | if (row.webhook_token === null) return undefined; |
| 389 | const base = origin?.relayHookBaseUrl ?? null; |
| 390 | return { |
| 391 | path: webhookPath(row.task_id, row.webhook_token), |
| 392 | url: base === null ? null : `${base}/${encodeURIComponent(row.task_id)}/${row.webhook_token}`, |
| 393 | hasSecret: row.webhook_secret !== null, |
| 394 | }; |
| 395 | } |
| 396 | |
| 397 | // Runs per task row; list and lookup operations already own the span. |
| 398 | const decodeRow = Effect.fnUntraced( |
| 399 | function* (row: ScheduledTaskRow, origin: WebhookOrigin | null = null) { |
| 400 | const schedule = yield* decodeScheduleJson(row.schedule_json); |
| 401 | const webhook = schedule.type === "webhook" ? webhookEndpoint(row, origin) : undefined; |
| 402 | const workspaceStrategy = yield* decodeWorkspaceStrategyJson(row.workspace_strategy_json); |
| 403 | const modelSelection = yield* decodeModelSelectionJson(row.model_selection_json); |
| 404 | return yield* decodeTask({ |
| 405 | id: row.task_id, |
| 406 | title: row.title, |
| 407 | prompt: row.prompt, |
| 408 | enabled: row.enabled === 1, |
| 409 | schedule, |
| 410 | projectId: row.project_id, |
| 411 | threadId: row.thread_id, |
| 412 | threadMode: row.thread_mode, |
| 413 | workspaceStrategy, |
| 414 | modelSelection, |
| 415 | runtimeMode: row.runtime_mode, |
| 416 | interactionMode: row.interaction_mode, |
| 417 | createdBy: row.created_by, |
| 418 | creationSource: row.creation_source, |
| 419 | createdAt: row.created_at, |
| 420 | updatedAt: row.updated_at, |
| 421 | nextRunAt: row.next_run_at, |
| 422 | lastRunAt: row.last_run_at, |
| 423 | lastRunStatus: row.last_run_status, |
| 424 | lastRunError: row.last_run_error, |
| 425 | runCount: row.run_count, |
| 426 | ...(webhook === undefined ? {} : { webhook }), |
| 427 | }); |
| 428 | }, |
| 429 | (effect, row, _origin?) => |
| 430 | effect.pipe( |
| 431 | Effect.mapError((cause) => { |
| 432 | const taskId = decodeTaskId(row.task_id); |
| 433 | return taskError("Could not decode schedule task row.", { |
| 434 | ...(Option.isSome(taskId) ? { taskId: taskId.value } : {}), |
| 435 | cause, |
| 436 | }); |
| 437 | }), |
| 438 | ), |
| 439 | ); |
| 440 | |
| 441 | export const listDueTasks = Effect.fn("ScheduledTaskService.listDueTasks")(function* ( |
| 442 | now: DateTime.DateTime, |
| 443 | ) { |
| 444 | const sql = yield* SqlClient.SqlClient; |
| 445 | const rows = yield* sql<ScheduledTaskRow>` |
| 446 | SELECT * FROM scheduled_tasks |
| 447 | WHERE enabled = 1 AND next_run_at IS NOT NULL |
| 448 | AND next_run_at <= ${iso(now)} AND last_run_status <> 'running' |
| 449 | ORDER BY next_run_at ASC, task_id ASC |
| 450 | `.pipe(NodeSqliteClient.readOnly); |
| 451 | const tasks: ScheduledTask[] = []; |
| 452 | for (const row of rows) { |
| 453 | const decoded = yield* Effect.result(decodeRow(row)); |
| 454 | if (Result.isSuccess(decoded)) { |
| 455 | const task = decoded.success; |
| 456 | // next_run_at is freeform; skip unparseable values before they defect the poll. |
| 457 | if (task.nextRunAt === null || Option.isSome(DateTime.make(task.nextRunAt))) { |
| 458 | tasks.push(task); |
| 459 | } else { |
| 460 | yield* Effect.logWarning("Skipping schedule task row with invalid next_run_at", { |
| 461 | taskId: row.task_id, |
| 462 | }); |
| 463 | } |
| 464 | } else { |
| 465 | yield* Effect.logWarning("Skipping undecodable schedule task row", { |
| 466 | taskId: row.task_id, |
| 467 | cause: decoded.failure, |
| 468 | }); |
| 469 | } |
| 470 | } |
| 471 | return tasks; |
| 472 | }); |
| 473 | |
| 474 | export const layer = Layer.effect( |
| 475 | ScheduledTaskService, |
| 476 | Effect.gen(function* () { |
| 477 | const sql = yield* SqlClient.SqlClient; |
| 478 | const random = yield* SecureRandom.SecureRandom; |
| 479 | const digest = yield* Digest.Digest; |
| 480 | const mac = yield* Mac.Mac; |
| 481 | const threadLaunch = yield* ThreadLaunchService.ThreadLaunchService; |
| 482 | const threadManagement = yield* ThreadManagementService.ThreadManagementService; |
| 483 | const serviceScope = yield* Effect.scope; |
| 484 | const managedFolders = yield* ManagedProjectFolders.ManagedProjectFolders; |
| 485 | const secretRequests = yield* SecretRequests.SecretRequests; |
| 486 | const scheduler = yield* Scheduler.Scheduler; |
| 487 | const readWebhookOrigin = yield* ScheduledTaskWebhookOrigin; |
| 488 | const managedPolicy = yield* ManagedPolicy.ManagedPolicy; |
| 489 | const webhookPolicyOrganization = ManagedPolicy.deniedBy(managedPolicy, "webhookTasks"); |
| 490 | // Existing webhook tasks stay stored but report why they do not run. |
| 491 | const webhookPolicyBlock = |
| 492 | webhookPolicyOrganization === undefined |
| 493 | ? undefined |
| 494 | : managedPolicyDeniedMessage(webhookPolicyOrganization, "webhook tasks"); |
| 495 | const withPolicy = (task: ScheduledTask): ScheduledTask => |
| 496 | webhookPolicyBlock !== undefined && task.schedule.type === "webhook" |
| 497 | ? { ...task, blockedByPolicy: webhookPolicyBlock } |
| 498 | : task; |
| 499 | const webhookPermits = yield* Ref.make<ReadonlyMap<ScheduledTaskId, Semaphore.Semaphore>>( |
| 500 | new Map(), |
| 501 | ); |
| 502 | // Include creation time so late deliveries cannot decrement a recreated task's count. |
| 503 | const webhookQueued = yield* Ref.make<ReadonlyMap<string, number>>(new Map()); |
| 504 | const webhookRateWindows = yield* Ref.make<ReadonlyMap<ScheduledTaskId, RateWindow>>(new Map()); |
| 505 | const webhookRejectedRateWindows = yield* Ref.make<ReadonlyMap<ScheduledTaskId, RateWindow>>( |
| 506 | new Map(), |
| 507 | ); |
| 508 | const webhookDedupeLogWindows = yield* Ref.make< |
| 509 | ReadonlyMap<ScheduledTaskId, ReadonlyArray<number>> |
| 510 | >(new Map()); |
| 511 | const pendingInFlight = yield* Ref.make<ReadonlySet<string>>(new Set()); |
| 512 | const webhookFilteredLogWindows = yield* Ref.make< |
| 513 | ReadonlyMap<ScheduledTaskId, ReadonlyArray<number>> |
| 514 | >(new Map()); |
| 515 | const activeRuns = yield* Ref.make<ReadonlySet<ScheduledTaskId>>(new Set()); |
| 516 | // Notifications re-emit the full list; slow subscribers need only the newest dirty signal. |
| 517 | const changesPubSub = yield* PubSub.sliding<void>(1); |
| 518 | const dueTasks = DueHint.make(); |
| 519 | const pendingWebhooks = DueHint.make(); |
| 520 | // Every scheduled_tasks write that can change next_run_at, enabled or run state notifies. |
| 521 | const notifyChanged = dueTasks.invalidate.pipe( |
| 522 | Effect.andThen(PubSub.publish(changesPubSub, undefined)), |
| 523 | Effect.asVoid, |
| 524 | ); |
| 525 | |
| 526 | const selectAllRows = () => |
| 527 | sql<ScheduledTaskRow>` |
| 528 | SELECT |
| 529 | task_id, |
| 530 | title, |
| 531 | prompt, |
| 532 | enabled, |
| 533 | schedule_json, |
| 534 | project_id, |
| 535 | thread_id, |
| 536 | thread_mode, |
| 537 | workspace_strategy_json, |
| 538 | model_selection_json, |
| 539 | runtime_mode, |
| 540 | interaction_mode, |
| 541 | created_by, |
| 542 | creation_source, |
| 543 | created_at, |
| 544 | updated_at, |
| 545 | next_run_at, |
| 546 | last_run_at, |
| 547 | last_run_status, |
| 548 | last_run_error, |
| 549 | run_count, |
| 550 | webhook_token, |
| 551 | webhook_secret |
| 552 | FROM scheduled_tasks |
| 553 | -- Not updated_at: toggles, runs and edits rewrite it, and clients render |
| 554 | -- this order as-is, so rows would move under the pointer. |
| 555 | ORDER BY created_at DESC, task_id ASC |
| 556 | `.pipe(NodeSqliteClient.readOnly); |
| 557 | |
| 558 | const listRows = Effect.fn("ScheduledTaskService.listRows")(function* () { |
| 559 | const rows = yield* selectAllRows(); |
| 560 | const origin = yield* readWebhookOrigin; |
| 561 | return yield* Effect.forEach( |
| 562 | rows, |
| 563 | (row) => decodeRow(row, origin).pipe(Effect.map(withPolicy)), |
| 564 | { concurrency: 1 }, |
| 565 | ); |
| 566 | }); |
| 567 | |
| 568 | const getRows = (id: ScheduledTaskId) => |
| 569 | sql<ScheduledTaskRow>` |
| 570 | SELECT |
| 571 | task_id, |
| 572 | title, |
| 573 | prompt, |
| 574 | enabled, |
| 575 | schedule_json, |
| 576 | project_id, |
| 577 | thread_id, |
| 578 | thread_mode, |
| 579 | workspace_strategy_json, |
| 580 | model_selection_json, |
| 581 | runtime_mode, |
| 582 | interaction_mode, |
| 583 | created_by, |
| 584 | creation_source, |
| 585 | created_at, |
| 586 | updated_at, |
| 587 | next_run_at, |
| 588 | last_run_at, |
| 589 | last_run_status, |
| 590 | last_run_error, |
| 591 | run_count, |
| 592 | webhook_token, |
| 593 | webhook_secret |
| 594 | FROM scheduled_tasks |
| 595 | WHERE task_id = ${id} |
| 596 | `.pipe(NodeSqliteClient.readOnly); |
| 597 | |
| 598 | const findTask = Effect.fn("ScheduledTaskService.findTask")(function* (id: ScheduledTaskId) { |
| 599 | const rows = yield* getRows(id).pipe( |
| 600 | Effect.mapError((cause) => |
| 601 | taskError("Could not load schedule task.", { taskId: id, cause }), |
| 602 | ), |
| 603 | ); |
| 604 | const row = rows[0]; |
| 605 | if (row === undefined) return null; |
| 606 | return withPolicy(yield* decodeRow(row, yield* readWebhookOrigin)); |
| 607 | }); |
| 608 | |
| 609 | const findWebhookCredentials = (id: ScheduledTaskId) => |
| 610 | getRows(id).pipe( |
| 611 | Effect.map((rows) => |
| 612 | rows[0] === undefined |
| 613 | ? null |
| 614 | : { token: rows[0].webhook_token, secret: rows[0].webhook_secret }, |
| 615 | ), |
| 616 | Effect.mapError((cause) => |
| 617 | taskError("Could not load schedule task.", { taskId: id, cause }), |
| 618 | ), |
| 619 | ); |
| 620 | |
| 621 | const newWebhookToken = random.randomBytes(32).pipe( |
| 622 | Effect.map((bytes) => Buffer.from(bytes).toString("base64url")), |
| 623 | Effect.mapError((cause) => taskError("Could not generate webhook token.", { cause })), |
| 624 | ); |
| 625 | |
| 626 | const loadTask = Effect.fn("ScheduledTaskService.loadTask")(function* (id: ScheduledTaskId) { |
| 627 | const task = yield* findTask(id); |
| 628 | if (task === null) { |
| 629 | return yield* taskError("Schedule task not found.", { taskId: id }); |
| 630 | } |
| 631 | return task; |
| 632 | }); |
| 633 | |
| 634 | // Run transitions own run-state columns; settings saves must not overwrite them. |
| 635 | // Check existence in the write so edits cannot undo concurrent deletion. |
| 636 | const saveTask = ( |
| 637 | task: ScheduledTask, |
| 638 | requireExisting: boolean, |
| 639 | /** The schedule this save read, when it left the schedule unchanged. */ |
| 640 | keptSchedule: ScheduledTask["schedule"] | null, |
| 641 | /** The thread this save unbinds; any other stored thread is kept. */ |
| 642 | unboundThreadId: ThreadId | null, |
| 643 | webhook: { |
| 644 | readonly token: string | null; |
| 645 | readonly secret: string | null; |
| 646 | readonly secretChanged: boolean; |
| 647 | }, |
| 648 | ) => |
| 649 | sql<{ task_id: string }>` |
| 650 | INSERT INTO scheduled_tasks ( |
| 651 | task_id, |
| 652 | title, |
| 653 | prompt, |
| 654 | enabled, |
| 655 | schedule_json, |
| 656 | project_id, |
| 657 | thread_id, |
| 658 | thread_mode, |
| 659 | workspace_strategy_json, |
| 660 | model_selection_json, |
| 661 | runtime_mode, |
| 662 | interaction_mode, |
| 663 | created_by, |
| 664 | creation_source, |
| 665 | created_at, |
| 666 | updated_at, |
| 667 | next_run_at, |
| 668 | last_run_at, |
| 669 | last_run_status, |
| 670 | last_run_error, |
| 671 | run_count, |
| 672 | webhook_token, |
| 673 | webhook_secret |
| 674 | ) |
| 675 | SELECT |
| 676 | ${task.id}, |
| 677 | ${task.title}, |
| 678 | ${task.prompt}, |
| 679 | ${task.enabled ? 1 : 0}, |
| 680 | ${JSON.stringify(task.schedule)}, |
| 681 | ${task.projectId}, |
| 682 | ${task.threadId}, |
| 683 | ${scheduledTaskThreadMode(task)}, |
| 684 | ${JSON.stringify(task.workspaceStrategy)}, |
| 685 | ${JSON.stringify(task.modelSelection)}, |
| 686 | ${task.runtimeMode}, |
| 687 | ${task.interactionMode}, |
| 688 | ${task.createdBy}, |
| 689 | ${task.creationSource}, |
| 690 | ${task.createdAt}, |
| 691 | ${task.updatedAt}, |
| 692 | ${task.nextRunAt}, |
| 693 | ${task.lastRunAt}, |
| 694 | ${task.lastRunStatus}, |
| 695 | ${task.lastRunError}, |
| 696 | ${task.runCount}, |
| 697 | ${webhook.token}, |
| 698 | ${webhook.secret} |
| 699 | WHERE ${requireExisting ? 0 : 1} = 1 |
| 700 | OR EXISTS (SELECT 1 FROM scheduled_tasks WHERE task_id = ${task.id}) |
| 701 | ON CONFLICT (task_id) |
| 702 | DO UPDATE SET |
| 703 | title = excluded.title, |
| 704 | prompt = excluded.prompt, |
| 705 | enabled = excluded.enabled, |
| 706 | schedule_json = excluded.schedule_json, |
| 707 | project_id = excluded.project_id, |
| 708 | -- A first run may bind the thread while this save is in flight, so a save |
| 709 | -- without a thread keeps any thread but the one it unbinds. |
| 710 | thread_id = CASE |
| 711 | WHEN excluded.thread_mode = 'new_thread' THEN NULL |
| 712 | WHEN excluded.project_id != scheduled_tasks.project_id THEN excluded.thread_id |
| 713 | ELSE COALESCE(excluded.thread_id, NULLIF(scheduled_tasks.thread_id, ${unboundThreadId})) |
| 714 | END, |
| 715 | thread_mode = excluded.thread_mode, |
| 716 | workspace_strategy_json = excluded.workspace_strategy_json, |
| 717 | model_selection_json = excluded.model_selection_json, |
| 718 | runtime_mode = excluded.runtime_mode, |
| 719 | interaction_mode = excluded.interaction_mode, |
| 720 | creation_source = excluded.creation_source, |
| 721 | updated_at = excluded.updated_at, |
| 722 | -- Keep a next_run_at that a finishing run moved during this save, unless |
| 723 | -- another edit changed the schedule or enabled state meanwhile. |
| 724 | next_run_at = CASE |
| 725 | WHEN ${keptSchedule === null ? 0 : 1} = 1 |
| 726 | AND scheduled_tasks.schedule_json IN ( |
| 727 | excluded.schedule_json, |
| 728 | ${keptSchedule === null ? null : JSON.stringify(keptSchedule)} |
| 729 | ) |
| 730 | AND scheduled_tasks.enabled = excluded.enabled |
| 731 | THEN scheduled_tasks.next_run_at |
| 732 | ELSE excluded.next_run_at |
| 733 | END, |
| 734 | -- Only rotate changes a live token, so a save racing a rotation |
| 735 | -- cannot bring the old URL back. |
| 736 | webhook_token = CASE |
| 737 | WHEN excluded.webhook_token IS NULL THEN NULL |
| 738 | ELSE COALESCE(scheduled_tasks.webhook_token, excluded.webhook_token) |
| 739 | END, |
| 740 | webhook_secret = CASE |
| 741 | WHEN ${webhook.secretChanged ? 1 : 0} = 1 THEN excluded.webhook_secret |
| 742 | ELSE scheduled_tasks.webhook_secret |
| 743 | END |
| 744 | RETURNING task_id |
| 745 | `.pipe( |
| 746 | Effect.mapError((cause) => |
| 747 | taskError("Could not save schedule task.", { taskId: task.id, cause }), |
| 748 | ), |
| 749 | Effect.flatMap((rows) => |
| 750 | rows.length > 0 |
| 751 | ? Effect.void |
| 752 | : taskError("Schedule task not found.", { taskId: task.id }), |
| 753 | ), |
| 754 | ); |
| 755 | |
| 756 | const deleteRow = (id: ScheduledTaskId) => |
| 757 | sql |
| 758 | .withTransaction( |
| 759 | sql`DELETE FROM scheduled_task_webhook_deliveries WHERE task_id = ${id}`.pipe( |
| 760 | Effect.andThen(sql`DELETE FROM scheduled_tasks WHERE task_id = ${id}`), |
| 761 | ), |
| 762 | ) |
| 763 | .pipe( |
| 764 | Effect.mapError((cause) => |
| 765 | taskError("Could not delete schedule task.", { taskId: id, cause }), |
| 766 | ), |
| 767 | ); |
| 768 | |
| 769 | const markRunning = (id: ScheduledTaskId, startedAtIso: string, webhookDeliveryId?: string) => |
| 770 | sql<{ task_id: string }>` |
| 771 | UPDATE scheduled_tasks |
| 772 | SET updated_at = ${startedAtIso}, |
| 773 | last_run_at = ${startedAtIso}, |
| 774 | last_run_webhook_delivery_id = ${webhookDeliveryId ?? null}, |
| 775 | last_run_status = 'running', |
| 776 | last_run_error = NULL |
| 777 | WHERE task_id = ${id} |
| 778 | RETURNING task_id |
| 779 | `.pipe( |
| 780 | Effect.mapError((cause) => |
| 781 | taskError("Could not mark schedule task as running.", { taskId: id, cause }), |
| 782 | ), |
| 783 | Effect.flatMap((rows) => |
| 784 | rows.length > 0 ? Effect.void : taskError("Schedule task not found.", { taskId: id }), |
| 785 | ), |
| 786 | ); |
| 787 | |
| 788 | const markCompleted = (input: { |
| 789 | readonly id: ScheduledTaskId; |
| 790 | readonly completedAtIso: string; |
| 791 | readonly nextRunAtIso: string | null; |
| 792 | readonly status: "succeeded" | "failed"; |
| 793 | readonly error: string | null; |
| 794 | readonly startedAtIso: string; |
| 795 | }) => |
| 796 | sql` |
| 797 | UPDATE scheduled_tasks |
| 798 | SET updated_at = ${input.completedAtIso}, |
| 799 | next_run_at = ${input.nextRunAtIso}, |
| 800 | last_run_status = ${input.status}, |
| 801 | last_run_error = ${input.error}, |
| 802 | run_count = run_count + 1 |
| 803 | WHERE task_id = ${input.id} |
| 804 | AND last_run_status = 'running' |
| 805 | AND last_run_at = ${input.startedAtIso} |
| 806 | RETURNING task_id |
| 807 | `.pipe( |
| 808 | Effect.map((rows) => rows.length > 0), |
| 809 | Effect.mapError((cause) => |
| 810 | taskError("Could not record schedule task run.", { taskId: input.id, cause }), |
| 811 | ), |
| 812 | ); |
| 813 | |
| 814 | // A failure between markRunning and markCompleted must still write a terminal record: |
| 815 | // runDueTasks skips 'running' rows forever, and the dispatch may already have gone out, |
| 816 | // so next_run_at must advance. Pending webhook attempts count only with their completion. |
| 817 | const releaseStuckRun = Effect.fn("ScheduledTaskService.releaseStuckRun")( |
| 818 | function* ( |
| 819 | task: ScheduledTask, |
| 820 | startedAtIso: string, |
| 821 | message: string, |
| 822 | pendingDeliveryId?: string, |
| 823 | ) { |
| 824 | const now = yield* localNow; |
| 825 | const reread = yield* Effect.result(findTask(task.id)); |
| 826 | if (Result.isSuccess(reread) && reread.success === null) return; |
| 827 | const source = Result.isSuccess(reread) && reread.success !== null ? reread.success : task; |
| 828 | yield* sql` |
| 829 | UPDATE scheduled_tasks |
| 830 | SET last_run_status = 'failed', |
| 831 | last_run_error = ${message}, |
| 832 | next_run_at = ${nextRunAfterRun({ ...source, lastRunAt: startedAtIso }, now)}, |
| 833 | updated_at = ${iso(now)}, |
| 834 | run_count = run_count + CASE WHEN EXISTS ( |
| 835 | SELECT 1 FROM scheduled_task_webhook_pending |
| 836 | WHERE delivery_id = ${pendingDeliveryId ?? null} AND state = 'running' |
| 837 | ) THEN 0 ELSE 1 END |
| 838 | WHERE task_id = ${task.id} AND last_run_status = 'running' |
| 839 | `; |
| 840 | yield* notifyChanged; |
| 841 | }, |
| 842 | (effect, task, _startedAtIso, _message, _pendingDeliveryId) => |
| 843 | effect.pipe( |
| 844 | Effect.catch((cause) => |
| 845 | Effect.logWarning("Could not release stuck schedule task run", { |
| 846 | taskId: task.id, |
| 847 | cause, |
| 848 | }), |
| 849 | ), |
| 850 | ), |
| 851 | ); |
| 852 | |
| 853 | // Reads every run row but only the queued runs' messages, so the cost per |
| 854 | // due interval tracks the thread's run count, not its transcript. |
| 855 | const queuedOccurrence = Effect.fn("ScheduledTaskService.queuedOccurrence")(function* ( |
| 856 | taskId: ScheduledTaskId, |
| 857 | threadId: ThreadId, |
| 858 | ) { |
| 859 | const { runs } = yield* threadManagement.getThreadRecords(threadId, ["runs"]); |
| 860 | const queued = runs.filter((run) => run.status === "queued"); |
| 861 | if (queued.length === 0) return null; |
| 862 | const queuedMessageIds = new Set(queued.map((run) => run.userMessageId)); |
| 863 | const { messages } = yield* threadManagement.getThreadRecords(threadId, ["messages"], { |
| 864 | messageIds: [...queuedMessageIds], |
| 865 | }); |
| 866 | const taskMessageIds = new Set( |
| 867 | messages |
| 868 | .filter((message) => message.scheduledTaskId === taskId) |
| 869 | .map((message) => message.id), |
| 870 | ); |
| 871 | const occurrences = queued.filter((run) => taskMessageIds.has(run.userMessageId)); |
| 872 | return occurrences.some((run) => run.queueHeld === true) |
| 873 | ? "held" |
| 874 | : occurrences.length > 0 |
| 875 | ? "queued" |
| 876 | : null; |
| 877 | }); |
| 878 | |
| 879 | // The run already started the thread, so a failed write only costs the binding. |
| 880 | // A thread deleted before the binding landed already ran its task cleanup, so apply that |
| 881 | // cleanup here; a later deletion finds the binding. An unconfirmed lookup keeps the binding. |
| 882 | const bindThread = Effect.fn("ScheduledTaskService.bindThread")( |
| 883 | function* (task: ScheduledTask, threadId: ThreadId) { |
| 884 | yield* sql` |
| 885 | UPDATE scheduled_tasks SET thread_id = ${threadId} |
| 886 | WHERE task_id = ${task.id} AND created_at = ${task.createdAt} |
| 887 | AND project_id = ${task.projectId} |
| 888 | AND thread_id IS NULL AND thread_mode != 'new_thread' |
| 889 | `; |
| 890 | const shell = yield* threadManagement.getThreadShell(threadId); |
| 891 | if (shell === null || shell.deletedAt != null) yield* deleteForThread(threadId); |
| 892 | }, |
| 893 | Effect.catchCause((cause) => |
| 894 | Effect.logWarning("Could not bind a scheduled task to its first thread.", cause), |
| 895 | ), |
| 896 | ); |
| 897 | |
| 898 | // A dispatch is recorded once the message is accepted; the provider can still |
| 899 | // reject the turn after that, so follow the run and record its failure. A one-time |
| 900 | // task whose turn completed at or after its runAt is spent, so it is removed; its |
| 901 | // thread stays. A paused task always has a NULL next_run_at, so that alone proves nothing. |
| 902 | const recordRunOutcome = Effect.fn("ScheduledTaskService.recordRunOutcome")(function* ( |
| 903 | taskId: ScheduledTaskId, |
| 904 | startedAtIso: string, |
| 905 | threadId: ThreadId, |
| 906 | messageId: MessageId, |
| 907 | ) { |
| 908 | const shell = yield* threadManagement.getThreadShell(threadId); |
| 909 | if (shell === null) return; |
| 910 | const thread = { projectId: shell.projectId, threadId }; |
| 911 | const { runs } = yield* threadManagement.getProjectThreadRecords(thread, ["runs"]); |
| 912 | const dispatched = runs.find((run) => run.userMessageId === messageId); |
| 913 | if (dispatched === undefined) return; |
| 914 | const { run } = yield* threadManagement.waitForThread({ |
| 915 | ...thread, |
| 916 | runId: dispatched.id, |
| 917 | timeoutMs: RUN_OUTCOME_WAIT_MS, |
| 918 | }); |
| 919 | if (run === null || !ThreadManagementService.isTerminalRunStatus(run.status)) return; |
| 920 | if (run.status === "completed") { |
| 921 | const removed = yield* sql` |
| 922 | DELETE FROM scheduled_tasks |
| 923 | WHERE task_id = ${taskId} AND last_run_at = ${startedAtIso} |
| 924 | AND last_run_status = 'succeeded' AND next_run_at IS NULL |
| 925 | AND json_extract(schedule_json, '$.type') = 'once' |
| 926 | AND julianday(last_run_at) >= julianday(json_extract(schedule_json, '$.runAt')) |
| 927 | RETURNING task_id |
| 928 | `; |
| 929 | if (removed.length > 0) yield* notifyChanged; |
| 930 | return; |
| 931 | } |
| 932 | // Only the run's failed errors feed latestRootProviderFailure; a long-lived task |
| 933 | // thread holds tens of thousands of items. |
| 934 | const { turnItems, messages } = yield* threadManagement.getProjectThreadRecords( |
| 935 | thread, |
| 936 | ["turnItems", "messages"], |
| 937 | { |
| 938 | messageIds: [messageId], |
| 939 | turnItemRunId: run.id, |
| 940 | turnItemTypes: ["error"], |
| 941 | turnItemStatuses: ["failed"], |
| 942 | }, |
| 943 | ); |
| 944 | // A queued message promoted to a steer cancels its run and joins the active one. |
| 945 | if ( |
| 946 | run.status === "cancelled" && |
| 947 | messages.find((message) => message.id === messageId)?.runId !== run.id |
| 948 | ) |
| 949 | return; |
| 950 | const error = |
| 951 | latestRootProviderFailure(run, turnItems)?.message ?? UNSUCCESSFUL_RUN_ERRORS[run.status]; |
| 952 | const updated = yield* sql` |
| 953 | UPDATE scheduled_tasks |
| 954 | SET last_run_status = 'failed', last_run_error = ${error} |
| 955 | WHERE task_id = ${taskId} AND last_run_at = ${startedAtIso} |
| 956 | AND (last_run_status = 'succeeded' |
| 957 | OR (last_run_status = 'failed' AND last_run_error = ${HELD_OCCURRENCE_ERROR})) |
| 958 | RETURNING task_id |
| 959 | `; |
| 960 | if (updated.length > 0) yield* notifyChanged; |
| 961 | }); |
| 962 | |
| 963 | const runTask = Effect.fn("ScheduledTaskService.runTask")(function* ( |
| 964 | task: ScheduledTask, |
| 965 | trigger: "scheduled" | "manual" | "webhook", |
| 966 | webhook?: { |
| 967 | readonly deliveryId: string; |
| 968 | readonly prompt: string; |
| 969 | readonly dispatched: Ref.Ref<boolean>; |
| 970 | }, |
| 971 | ) { |
| 972 | // Read before reserving: nothing between the reservation and its release may fail. |
| 973 | const startedAt = yield* localNow; |
| 974 | const startedAtIso = iso(startedAt); |
| 975 | const run = Effect.gen(function* () { |
| 976 | // Re-read stale poll snapshots so deleted, paused or postponed tasks cannot fire. |
| 977 | const active = yield* findTask(task.id); |
| 978 | if (active === null) { |
| 979 | if (webhook !== undefined) { |
| 980 | return yield* new WebhookDeliverySkipped({ |
| 981 | reason: "The task was deleted before this delivery ran.", |
| 982 | }); |
| 983 | } |
| 984 | if (trigger !== "scheduled") { |
| 985 | return yield* taskError("Schedule task not found.", { taskId: task.id }); |
| 986 | } |
| 987 | return { task, threadId: null, dispatchFailed: false }; |
| 988 | } |
| 989 | if ( |
| 990 | webhook !== undefined && |
| 991 | (yield* sql`SELECT 1 FROM scheduled_task_webhook_pending WHERE delivery_id = ${webhook.deliveryId} AND state = 'completed'`.pipe( |
| 992 | NodeSqliteClient.readOnly, |
| 993 | )).length > 0 |
| 994 | ) |
| 995 | return { |
| 996 | task: { ...active, lastRunStatus: "succeeded" as const }, |
| 997 | threadId: null, |
| 998 | dispatchFailed: false, |
| 999 | }; |
| 1000 | const parsedNextRunAt = |
| 1001 | active.nextRunAt === null ? Option.none() : DateTime.make(active.nextRunAt); |
| 1002 | if ( |
| 1003 | trigger === "scheduled" && |
| 1004 | (!active.enabled || |
| 1005 | Option.isNone(parsedNextRunAt) || |
| 1006 | DateTime.toEpochMillis(parsedNextRunAt.value) > DateTime.toEpochMillis(startedAt)) |
| 1007 | ) { |
| 1008 | return { task: active, threadId: null, dispatchFailed: false }; |
| 1009 | } |
| 1010 | // A thread that stays busy would otherwise collect one copy of the |
| 1011 | // prompt per interval. A failed read falls through to the normal send. |
| 1012 | if ( |
| 1013 | trigger === "scheduled" && |
| 1014 | active.schedule.type !== "once" && |
| 1015 | active.threadId !== null |
| 1016 | ) { |
| 1017 | const occurrence = yield* queuedOccurrence( |
| 1018 | active.id, |
| 1019 | ThreadId.make(active.threadId), |
| 1020 | ).pipe(Effect.orElseSucceed(() => null)); |
| 1021 | if (occurrence !== null) { |
| 1022 | yield* rescheduleMissedRun( |
| 1023 | active, |
| 1024 | startedAt, |
| 1025 | "previous occurrence still queued", |
| 1026 | occurrence === "held" ? HELD_OCCURRENCE_ERROR : undefined, |
| 1027 | ); |
| 1028 | return { task: active, threadId: null, dispatchFailed: false }; |
| 1029 | } |
| 1030 | } |
| 1031 | // Reject queued deliveries after pause, same-id recreation or trigger changes. |
| 1032 | if (webhook !== undefined) { |
| 1033 | const reason = |
| 1034 | active.createdAt !== task.createdAt |
| 1035 | ? "The task was replaced before this delivery ran." |
| 1036 | : active.schedule.type !== "webhook" |
| 1037 | ? "The task's trigger changed before this delivery ran." |
| 1038 | : webhookPolicyBlock !== undefined |
| 1039 | ? webhookPolicyBlock |
| 1040 | : !active.enabled |
| 1041 | ? "The task was paused before this delivery ran." |
| 1042 | : null; |
| 1043 | if (reason !== null) return yield* new WebhookDeliverySkipped({ reason }); |
| 1044 | } |
| 1045 | |
| 1046 | if (webhook === undefined) yield* markRunning(active.id, startedAtIso); |
| 1047 | else |
| 1048 | yield* sql |
| 1049 | .withTransaction( |
| 1050 | Effect.gen(function* () { |
| 1051 | yield* markRunning(active.id, startedAtIso, webhook.deliveryId); |
| 1052 | // Recovery must distinguish this delivery from other windows waiting on the same task. |
| 1053 | yield* sql`UPDATE scheduled_task_webhook_pending SET state = 'running' WHERE delivery_id = ${webhook.deliveryId}`; |
| 1054 | }), |
| 1055 | ) |
| 1056 | .pipe( |
| 1057 | Effect.mapError((cause) => |
| 1058 | taskError("Could not start webhook run.", { taskId: task.id, cause }), |
| 1059 | ), |
| 1060 | ); |
| 1061 | |
| 1062 | yield* notifyChanged; |
| 1063 | |
| 1064 | const fireKey = |
| 1065 | webhook === undefined |
| 1066 | ? `${active.id}:${DateTime.toEpochMillis(startedAt)}:${trigger}` |
| 1067 | : `${active.id}:webhook:${webhook.deliveryId}`; |
| 1068 | const commandId = CommandId.make(`scheduled-task:${fireKey}`); |
| 1069 | const messageId = MessageId.make(`scheduled-task-message:${fireKey}`); |
| 1070 | const prompt = webhook?.prompt ?? active.prompt; |
| 1071 | const threadMode = scheduledTaskThreadMode(active); |
| 1072 | |
| 1073 | // A bound thread can move to another project (No project to a project) while |
| 1074 | // this fires. Follow it; one move needs at most two retries. Same ids replay safely. |
| 1075 | const sendBound = ( |
| 1076 | threadId: ThreadId, |
| 1077 | projectId: ProjectId, |
| 1078 | retries: number, |
| 1079 | ): ReturnType<typeof threadManagement.sendToThread> => |
| 1080 | threadManagement |
| 1081 | .sendToThread({ |
| 1082 | projectId, |
| 1083 | commandId, |
| 1084 | threadId, |
| 1085 | messageId, |
| 1086 | scheduledTaskId: active.id, |
| 1087 | text: prompt, |
| 1088 | attachments: [], |
| 1089 | mode: "queue", |
| 1090 | ...(threadMode === "fresh_session" ? { freshContext: true } : {}), |
| 1091 | createdBy: active.createdBy, |
| 1092 | creationSource: active.creationSource, |
| 1093 | }) |
| 1094 | .pipe( |
| 1095 | Effect.catchTags({ |
| 1096 | ThreadManagementThreadNotFoundError: (error) => |
| 1097 | retries === 0 |
| 1098 | ? Effect.fail(error) |
| 1099 | : threadManagement.getThreadShell(threadId).pipe( |
| 1100 | Effect.orElseSucceed(() => null), |
| 1101 | Effect.flatMap((shell) => |
| 1102 | shell === null || shell.projectId === projectId |
| 1103 | ? Effect.fail(error) |
| 1104 | : sendBound(threadId, shell.projectId, retries - 1), |
| 1105 | ), |
| 1106 | ), |
| 1107 | }), |
| 1108 | ); |
| 1109 | |
| 1110 | // Capture defects and interrupts too so dispatch failures still reach markCompleted. |
| 1111 | const result = |
| 1112 | active.threadId === null |
| 1113 | ? yield* Effect.exit( |
| 1114 | threadLaunch |
| 1115 | .launch({ |
| 1116 | commandId, |
| 1117 | projectId: active.projectId, |
| 1118 | title: active.title, |
| 1119 | modelSelection: active.modelSelection, |
| 1120 | runtimeMode: active.runtimeMode, |
| 1121 | interactionMode: active.interactionMode, |
| 1122 | workspaceStrategy: active.workspaceStrategy, |
| 1123 | initialMessage: { |
| 1124 | messageId, |
| 1125 | scheduledTaskId: active.id, |
| 1126 | text: prompt, |
| 1127 | attachments: [], |
| 1128 | }, |
| 1129 | createdBy: active.createdBy, |
| 1130 | creationSource: active.creationSource, |
| 1131 | }) |
| 1132 | .pipe( |
| 1133 | Effect.map((launched) => launched.threadId), |
| 1134 | Effect.tap((threadId) => |
| 1135 | threadMode === "new_thread" ? Effect.void : bindThread(active, threadId), |
| 1136 | ), |
| 1137 | ), |
| 1138 | ) |
| 1139 | : yield* Effect.exit( |
| 1140 | // The stored selection only seeds fresh threads. A bound run |
| 1141 | // follows the thread's current provider, so it passes none. |
| 1142 | sendBound(ThreadId.make(active.threadId), active.projectId, 2).pipe( |
| 1143 | Effect.as(ThreadId.make(active.threadId)), |
| 1144 | ), |
| 1145 | ); |
| 1146 | |
| 1147 | const runSucceeded = result._tag === "Success"; |
| 1148 | if (webhook !== undefined && runSucceeded) yield* Ref.set(webhook.dispatched, true); |
| 1149 | const completedAt = yield* localNow; |
| 1150 | const lastRunStatus = runSucceeded ? ("succeeded" as const) : ("failed" as const); |
| 1151 | const lastRunError = runSucceeded ? null : dispatchErrorMessage(result.cause); |
| 1152 | const threadId = runSucceeded ? result.value : null; |
| 1153 | // One transaction, so an edit cannot land between reading the schedule and |
| 1154 | // writing the next run computed from it. The origin is read first: it reads |
| 1155 | // files, and the transaction holds SQLite's writer lock. |
| 1156 | const origin = yield* readWebhookOrigin; |
| 1157 | const record = Effect.gen(function* () { |
| 1158 | const [row] = yield* getRows(task.id); |
| 1159 | const current = row === undefined ? null : yield* decodeRow(row, origin); |
| 1160 | const scheduleSource = current ?? task; |
| 1161 | const completed: ScheduledTask = { |
| 1162 | ...scheduleSource, |
| 1163 | updatedAt: iso(completedAt), |
| 1164 | lastRunAt: startedAtIso, |
| 1165 | // A manual run that starts before a one-time run must not use it up. |
| 1166 | nextRunAt: nextRunAt( |
| 1167 | scheduleSource, |
| 1168 | scheduleSource.schedule.type === "once" ? startedAt : completedAt, |
| 1169 | ), |
| 1170 | lastRunStatus, |
| 1171 | lastRunError, |
| 1172 | runCount: scheduleSource.runCount + 1, |
| 1173 | }; |
| 1174 | if (current === null) return { recorded: completed, written: false }; |
| 1175 | // startedAtIso in the guard ensures this writes only to the row this |
| 1176 | // run marked as running — a task deleted mid-run and recreated with |
| 1177 | // the same id (idempotent commandId replay) must not be stamped. |
| 1178 | const completion = markCompleted({ |
| 1179 | id: task.id, |
| 1180 | completedAtIso: completed.updatedAt, |
| 1181 | nextRunAtIso: completed.nextRunAt, |
| 1182 | status: lastRunStatus, |
| 1183 | error: lastRunError, |
| 1184 | startedAtIso, |
| 1185 | }); |
| 1186 | let written = false; |
| 1187 | if (webhook === undefined) written = yield* completion; |
| 1188 | else { |
| 1189 | const pending = |
| 1190 | yield* sql`SELECT 1 FROM scheduled_task_webhook_pending WHERE delivery_id = ${webhook.deliveryId}`; |
| 1191 | if (pending.length === 0) written = yield* completion; |
| 1192 | else { |
| 1193 | const counted = |
| 1194 | yield* sql`UPDATE scheduled_task_webhook_pending SET state = ${runSucceeded ? "completed" : "failed"} |
| 1195 | WHERE delivery_id = ${webhook.deliveryId} AND state NOT IN ('completed', 'failed') RETURNING delivery_id`; |
| 1196 | if (counted.length > 0) written = yield* completion; |
| 1197 | } |
| 1198 | } |
| 1199 | // A row recreated mid-run is not this run's, so report what is stored. |
| 1200 | return { recorded: written ? completed : current, written }; |
| 1201 | }); |
| 1202 | const { recorded, written } = yield* sql |
| 1203 | .withTransaction(record) |
| 1204 | .pipe( |
| 1205 | Effect.mapError((cause) => |
| 1206 | isScheduledTaskError(cause) |
| 1207 | ? cause |
| 1208 | : taskError( |
| 1209 | webhook === undefined |
| 1210 | ? "Could not record schedule task run." |
| 1211 | : "Could not record webhook completion.", |
| 1212 | { taskId: task.id, cause }, |
| 1213 | ), |
| 1214 | ), |
| 1215 | ); |
| 1216 | if (written) yield* notifyChanged; |
| 1217 | if (written && threadId !== null) { |
| 1218 | yield* recordRunOutcome(active.id, startedAtIso, threadId, messageId).pipe( |
| 1219 | Effect.catchCause((cause) => |
| 1220 | Effect.logWarning("Could not record scheduled run outcome", { |
| 1221 | taskId: active.id, |
| 1222 | cause, |
| 1223 | }), |
| 1224 | ), |
| 1225 | Effect.forkIn(serviceScope), |
| 1226 | ); |
| 1227 | } |
| 1228 | // The stored row can belong to a replacement, so report this dispatch separately. |
| 1229 | return { task: recorded, threadId, dispatchFailed: !runSucceeded }; |
| 1230 | }).pipe( |
| 1231 | Effect.onError((cause) => |
| 1232 | releaseStuckRun(task, startedAtIso, errorMessage(cause), webhook?.deliveryId), |
| 1233 | ), |
| 1234 | ); |
| 1235 | |
| 1236 | // Reserve and release as one bracket so an interrupt cannot leave the task marked busy. |
| 1237 | return yield* Effect.acquireUseRelease( |
| 1238 | Ref.modify(activeRuns, (active) => { |
| 1239 | if (active.has(task.id)) return [false, active] as const; |
| 1240 | const next = new Set(active); |
| 1241 | next.add(task.id); |
| 1242 | return [true, next] as const; |
| 1243 | }), |
| 1244 | (reserved) => |
| 1245 | reserved |
| 1246 | ? run |
| 1247 | : trigger !== "scheduled" |
| 1248 | ? Effect.fail(taskError("Schedule task is already running.", { taskId: task.id })) |
| 1249 | : Effect.succeed({ task, threadId: null, dispatchFailed: false }), |
| 1250 | (reserved) => |
| 1251 | reserved |
| 1252 | ? Ref.update(activeRuns, (active) => { |
| 1253 | const next = new Set(active); |
| 1254 | next.delete(task.id); |
| 1255 | return next; |
| 1256 | }) |
| 1257 | : Effect.void, |
| 1258 | ); |
| 1259 | }); |
| 1260 | |
| 1261 | const rescheduleMissedRun = Effect.fn("ScheduledTaskService.rescheduleMissedRun")(function* ( |
| 1262 | task: ScheduledTask, |
| 1263 | now: DateTime.DateTime, |
| 1264 | reason = "missed", |
| 1265 | runError?: string, |
| 1266 | ) { |
| 1267 | const next = nextRunAt(task, now); |
| 1268 | yield* Effect.logInfo("Skipping schedule task run", { |
| 1269 | taskId: task.id, |
| 1270 | reason, |
| 1271 | missedRunAt: task.nextRunAt, |
| 1272 | rescheduledTo: next, |
| 1273 | }); |
| 1274 | // `task` can be a stale batch snapshot; a run or edit since then already moved the row on. |
| 1275 | const updated = yield* sql` |
| 1276 | UPDATE scheduled_tasks |
| 1277 | SET next_run_at = ${next}, |
| 1278 | updated_at = ${iso(now)} |
| 1279 | ${ |
| 1280 | runError === undefined |
| 1281 | ? sql`` |
| 1282 | : sql`, last_run_status = CASE |
| 1283 | WHEN last_run_status IS ${task.lastRunStatus} AND last_run_error IS ${task.lastRunError} |
| 1284 | THEN 'failed' ELSE last_run_status END, |
| 1285 | last_run_error = CASE |
| 1286 | WHEN last_run_status IS ${task.lastRunStatus} AND last_run_error IS ${task.lastRunError} |
| 1287 | THEN ${runError} ELSE last_run_error END` |
| 1288 | } |
| 1289 | WHERE task_id = ${task.id} |
| 1290 | AND next_run_at IS ${task.nextRunAt} |
| 1291 | AND last_run_at IS ${task.lastRunAt} |
| 1292 | RETURNING task_id |
| 1293 | `.pipe( |
| 1294 | Effect.mapError((cause) => |
| 1295 | taskError("Could not reschedule missed schedule task run.", { taskId: task.id, cause }), |
| 1296 | ), |
| 1297 | ); |
| 1298 | if (updated.length > 0) yield* notifyChanged; |
| 1299 | }); |
| 1300 | |
| 1301 | const runDueTasks = Effect.fn("ScheduledTaskService.runDueTasks")(function* () { |
| 1302 | const now = yield* localNow; |
| 1303 | const nowMs = DateTime.toEpochMillis(now); |
| 1304 | if (yield* dueTasks.isIdle(nowMs)) return; |
| 1305 | const readVersion = yield* dueTasks.version; |
| 1306 | const [next] = yield* sql<{ next_run_at: string | null }>` |
| 1307 | SELECT MIN(next_run_at) AS next_run_at FROM scheduled_tasks |
| 1308 | WHERE enabled = 1 AND next_run_at IS NOT NULL AND last_run_status <> 'running' |
| 1309 | `.pipe( |
| 1310 | NodeSqliteClient.readOnly, |
| 1311 | Effect.mapError((cause) => taskError("Could not list schedule tasks.", { cause })), |
| 1312 | ); |
| 1313 | // Compare as text like listDueTasks: expanded years (`+011533-…`) sort first but parse far ahead. |
| 1314 | const nextRunAt = next?.next_run_at ?? null; |
| 1315 | if (nextRunAt === null || nextRunAt > iso(now)) { |
| 1316 | const nextDueAt = |
| 1317 | nextRunAt === null |
| 1318 | ? Option.some(Infinity) |
| 1319 | : DateTime.make(nextRunAt).pipe(Option.map(DateTime.toEpochMillis)); |
| 1320 | if (Option.isSome(nextDueAt) && nextDueAt.value > nowMs) { |
| 1321 | yield* dueTasks.record(readVersion, nextDueAt.value, nowMs); |
| 1322 | } |
| 1323 | return; |
| 1324 | } |
| 1325 | const tasks = yield* listDueTasks(now).pipe( |
| 1326 | Effect.mapError((cause) => taskError("Could not list schedule tasks.", { cause })), |
| 1327 | ); |
| 1328 | const due = tasks.flatMap((task) => |
| 1329 | task.nextRunAt === null ? [] : [{ task, dueAt: DateTime.makeUnsafe(task.nextRunAt) }], |
| 1330 | ); |
| 1331 | yield* Effect.forEach( |
| 1332 | due, |
| 1333 | ({ task, dueAt }) => |
| 1334 | Effect.suspend(() => |
| 1335 | isMissedFixedTimeRun(task.schedule, dueAt, now) |
| 1336 | ? rescheduleMissedRun( |
| 1337 | task, |
| 1338 | now, |
| 1339 | "missed", |
| 1340 | "Missed a scheduled run because the server was not running at that time.", |
| 1341 | ) |
| 1342 | : runTask(task, "scheduled"), |
| 1343 | ).pipe( |
| 1344 | Effect.catch((cause) => |
| 1345 | Effect.logWarning("Scheduled task run failed", { taskId: task.id, cause }), |
| 1346 | ), |
| 1347 | ), |
| 1348 | { concurrency: 1, discard: true }, |
| 1349 | ); |
| 1350 | }); |
| 1351 | |
| 1352 | // Recover crash-left running rows that the due-task filter would otherwise skip forever. |
| 1353 | yield* Effect.gen(function* () { |
| 1354 | const rows = yield* selectAllRows(); |
| 1355 | const stuck = rows.filter((row) => row.last_run_status === "running"); |
| 1356 | if (stuck.length === 0) return; |
| 1357 | const now = yield* localNow; |
| 1358 | yield* Effect.forEach( |
| 1359 | stuck, |
| 1360 | (row) => |
| 1361 | Effect.gen(function* () { |
| 1362 | const decoded = yield* Effect.result(decodeRow(row)); |
| 1363 | if (Result.isSuccess(decoded)) { |
| 1364 | yield* sql` |
| 1365 | UPDATE scheduled_tasks |
| 1366 | SET last_run_status = 'failed', |
| 1367 | last_run_error = 'Run was interrupted by a server restart.', |
| 1368 | next_run_at = ${nextRunAfterRun(decoded.success, now)}, |
| 1369 | updated_at = ${iso(now)}, |
| 1370 | run_count = run_count + CASE WHEN EXISTS ( |
| 1371 | SELECT 1 FROM scheduled_task_webhook_pending |
| 1372 | WHERE task_id IS ${row.task_id} AND state = 'running' |
| 1373 | AND delivery_id = scheduled_tasks.last_run_webhook_delivery_id |
| 1374 | ) THEN 0 ELSE 1 END |
| 1375 | WHERE task_id IS ${row.task_id} AND last_run_status = 'running' |
| 1376 | `; |
| 1377 | return; |
| 1378 | } |
| 1379 | yield* Effect.logWarning( |
| 1380 | "Recovering undecodable schedule task row without rescheduling", |
| 1381 | { taskId: row.task_id, cause: decoded.failure }, |
| 1382 | ); |
| 1383 | yield* sql` |
| 1384 | UPDATE scheduled_tasks |
| 1385 | SET last_run_status = 'failed', |
| 1386 | last_run_error = 'Run was interrupted by a server restart.', |
| 1387 | updated_at = ${iso(now)}, |
| 1388 | run_count = run_count + CASE WHEN EXISTS ( |
| 1389 | SELECT 1 FROM scheduled_task_webhook_pending |
| 1390 | WHERE task_id IS ${row.task_id} AND state = 'running' |
| 1391 | AND delivery_id = scheduled_tasks.last_run_webhook_delivery_id |
| 1392 | ) THEN 0 ELSE 1 END |
| 1393 | WHERE task_id IS ${row.task_id} AND last_run_status = 'running' |
| 1394 | `; |
| 1395 | }), |
| 1396 | { concurrency: 1, discard: true }, |
| 1397 | ); |
| 1398 | yield* notifyChanged; |
| 1399 | }).pipe( |
| 1400 | Effect.catch((cause) => |
| 1401 | Effect.logWarning("Could not reset interrupted schedule task runs", { cause }), |
| 1402 | ), |
| 1403 | ); |
| 1404 | |
| 1405 | yield* scheduler.register("scheduled-tasks", runDueTasks()); |
| 1406 | |
| 1407 | const list: ScheduledTaskService["Service"]["list"] = () => |
| 1408 | listRows().pipe( |
| 1409 | Effect.map((tasks) => ({ |
| 1410 | tasks, |
| 1411 | timeZone: DateTime.zoneToString(DateTime.zoneMakeLocal()), |
| 1412 | })), |
| 1413 | Effect.mapError((cause) => taskError("Could not list schedule tasks.", { cause })), |
| 1414 | ); |
| 1415 | |
| 1416 | const subscribeList: ScheduledTaskService["Service"]["subscribeList"] = () => |
| 1417 | Stream.unwrap( |
| 1418 | Effect.gen(function* () { |
| 1419 | // Subscribe before reading to buffer concurrent changes. |
| 1420 | const subscription = yield* PubSub.subscribe(changesPubSub); |
| 1421 | return Stream.concat( |
| 1422 | Stream.fromEffect(list()), |
| 1423 | Stream.fromSubscription(subscription).pipe(Stream.mapEffect(() => list())), |
| 1424 | ); |
| 1425 | }), |
| 1426 | ); |
| 1427 | |
| 1428 | const requireDestinationThread = Effect.fn("ScheduledTaskService.requireDestinationThread")( |
| 1429 | function* (threadId: ThreadId, projectId: ProjectId, taskId: ScheduledTaskId) { |
| 1430 | const thread = yield* threadManagement |
| 1431 | .getThreadShell(threadId) |
| 1432 | .pipe( |
| 1433 | Effect.mapError((cause) => |
| 1434 | taskError("Could not read the destination thread.", { taskId, cause }), |
| 1435 | ), |
| 1436 | ); |
| 1437 | if (thread === null || thread.deletedAt !== null) { |
| 1438 | return yield* taskError("That thread no longer exists.", { taskId }); |
| 1439 | } |
| 1440 | if (thread.archivedAt !== null) { |
| 1441 | return yield* taskError( |
| 1442 | "That thread is archived. Unarchive it or pick another destination.", |
| 1443 | { taskId }, |
| 1444 | ); |
| 1445 | } |
| 1446 | if (thread.projectId !== projectId) { |
| 1447 | return yield* taskError("That thread belongs to a different project.", { taskId }); |
| 1448 | } |
| 1449 | }, |
| 1450 | ); |
| 1451 | |
| 1452 | const upsert: ScheduledTaskService["Service"]["upsert"] = Effect.fn( |
| 1453 | "ScheduledTaskService.upsert", |
| 1454 | )(function* (input) { |
| 1455 | // A webhook task the policy blocks can still be edited while it is off. |
| 1456 | if (input.enabled && input.schedule.type === "webhook" && webhookPolicyBlock !== undefined) { |
| 1457 | return yield* taskError( |
| 1458 | webhookPolicyBlock, |
| 1459 | input.id === undefined ? {} : { taskId: input.id }, |
| 1460 | ); |
| 1461 | } |
| 1462 | const now = yield* localNow; |
| 1463 | const uuid = |
| 1464 | input.commandId === undefined |
| 1465 | ? yield* random |
| 1466 | .uuidV4() |
| 1467 | .pipe( |
| 1468 | Effect.mapError((cause) => |
| 1469 | taskError("Could not generate schedule task id.", { cause }), |
| 1470 | ), |
| 1471 | ) |
| 1472 | : null; |
| 1473 | const id = |
| 1474 | input.id ?? |
| 1475 | ScheduledTaskId.make( |
| 1476 | input.commandId ? `scheduled-task:${input.commandId}` : `scheduled-task:${uuid}`, |
| 1477 | ); |
| 1478 | // Look up by the *resolved* id so idempotent creates (commandId replays) |
| 1479 | // keep their run history, and so real load failures propagate instead |
| 1480 | // of silently resetting an existing row. |
| 1481 | const existingTask = yield* findTask(id); |
| 1482 | const threadMode = resolveThreadMode(input, existingTask); |
| 1483 | const keptThreadId = |
| 1484 | input.threadId ?? (input.unbindThread === true ? null : (existingTask?.threadId ?? null)); |
| 1485 | // A bound thread belongs to its project, so moving the task leaves it for a new one. |
| 1486 | const movedOffThread = |
| 1487 | existingTask !== null && |
| 1488 | existingTask.projectId !== input.projectId && |
| 1489 | keptThreadId === existingTask.threadId; |
| 1490 | const threadId = threadMode === "new_thread" || movedOffThread ? null : keptThreadId; |
| 1491 | // An unchanged destination is not rechecked, so a task on an archived thread stays editable. |
| 1492 | if (threadId !== null && threadId !== existingTask?.threadId) { |
| 1493 | yield* requireDestinationThread(threadId, input.projectId, id); |
| 1494 | } |
| 1495 | // Refused on save rather than at run time; a disabled task can still be saved, so one |
| 1496 | // stored above the ceiling can be turned off before its mode is lowered. A task bound to a |
| 1497 | // thread runs in that thread's mode, which the orchestrator checks instead. Checked before |
| 1498 | // the save consumes a webhook secret. |
| 1499 | const modeDenial = |
| 1500 | input.enabled && threadId === null |
| 1501 | ? managedRuntimeModeDenial(managedPolicy.summary, input.runtimeMode) |
| 1502 | : undefined; |
| 1503 | if (modeDenial !== undefined) return yield* taskError(modeDenial, { taskId: id }); |
| 1504 | // Keep the existing next_run_at when the schedule itself is untouched: |
| 1505 | // editing a title or prompt must not postpone (or resurrect) a due |
| 1506 | // run — only schedule/enabled changes restart the clock. |
| 1507 | const schedule: ScheduledTask["schedule"] = |
| 1508 | input.schedule.type === "webhook" |
| 1509 | ? { |
| 1510 | type: "webhook", |
| 1511 | signature: |
| 1512 | input.schedule.signature == null |
| 1513 | ? null |
| 1514 | : { |
| 1515 | header: input.schedule.signature.header.toLowerCase(), |
| 1516 | encoding: input.schedule.signature.encoding, |
| 1517 | prefix: input.schedule.signature.prefix, |
| 1518 | }, |
| 1519 | maxDeliveryAgeMinutes: input.schedule.maxDeliveryAgeMinutes ?? null, |
| 1520 | ...(input.schedule.dedupe === undefined |
| 1521 | ? existingTask?.schedule.type === "webhook" && |
| 1522 | existingTask.schedule.dedupe !== undefined |
| 1523 | ? { dedupe: existingTask.schedule.dedupe } |
| 1524 | : {} |
| 1525 | : { dedupe: input.schedule.dedupe }), |
| 1526 | ...webhookFilters( |
| 1527 | // Omitted keeps the stored filters, so a client that predates |
| 1528 | // them can re-save a task without clearing them. |
| 1529 | input.schedule.filters ?? |
| 1530 | (existingTask?.schedule.type === "webhook" |
| 1531 | ? existingTask.schedule.filters |
| 1532 | : undefined), |
| 1533 | ), |
| 1534 | } |
| 1535 | : input.schedule.type === "once" |
| 1536 | ? { type: "once", runAt: iso(DateTime.makeUnsafe(input.schedule.runAt)) } |
| 1537 | : input.schedule; |
| 1538 | const webhook = |
| 1539 | input.schedule.type === "webhook" |
| 1540 | ? yield* Effect.gen(function* () { |
| 1541 | const existing = existingTask === null ? null : yield* findWebhookCredentials(id); |
| 1542 | const token = existing?.token ?? (yield* newWebhookToken); |
| 1543 | const signature = input.schedule.type === "webhook" ? input.schedule.signature : null; |
| 1544 | // A secretRef is a value the user entered for an agent; this |
| 1545 | // save consumes it, so it cannot be used again. A replay of a |
| 1546 | // save that already used it keeps the secret that save stored. |
| 1547 | const replay = input.commandId !== undefined && existing?.secret != null; |
| 1548 | const fromRef = |
| 1549 | signature?.secretRef === undefined |
| 1550 | ? undefined |
| 1551 | : yield* secretRequests |
| 1552 | .consume({ ref: signature.secretRef, projectId: input.projectId }) |
| 1553 | .pipe( |
| 1554 | Effect.catch((error) => |
| 1555 | replay |
| 1556 | ? Effect.void |
| 1557 | : Effect.fail(taskError(error.message, { taskId: id })), |
| 1558 | ), |
| 1559 | ); |
| 1560 | // A ref, when given, is the only source: a plain secret sent |
| 1561 | // alongside it must not replace what a replayed save stored. |
| 1562 | const provided = signature?.secretRef === undefined ? signature?.secret : fromRef; |
| 1563 | const secret = signature == null ? null : (provided ?? existing?.secret ?? null); |
| 1564 | const secretChanged = |
| 1565 | signature == null || provided !== undefined || existing === null; |
| 1566 | if (signature != null && secret === null) { |
| 1567 | return yield* taskError("A webhook signature check needs a signing secret.", { |
| 1568 | taskId: id, |
| 1569 | }); |
| 1570 | } |
| 1571 | return { token, secret, secretChanged }; |
| 1572 | }) |
| 1573 | : { token: null, secret: null, secretChanged: true }; |
| 1574 | // A folder that is not a Git repository, Scratch included, cannot hold a worktree. |
| 1575 | const workspaceStrategy: ScheduledTask["workspaceStrategy"] = |
| 1576 | input.workspaceStrategy.type === "worktree" && |
| 1577 | ((yield* managedFolders.isScratchProject(input.projectId)) || |
| 1578 | (yield* threadLaunch.isNonGitProject(input.projectId))) |
| 1579 | ? { type: "root" } |
| 1580 | : input.workspaceStrategy; |
| 1581 | const scheduleUnchanged = |
| 1582 | existingTask !== null && |
| 1583 | existingTask.enabled === input.enabled && |
| 1584 | isSameSchedule(existingTask.schedule, schedule); |
| 1585 | if (!scheduleUnchanged) |
| 1586 | yield* requireFutureOnce({ enabled: input.enabled, schedule }, now, id); |
| 1587 | const task: ScheduledTask = { |
| 1588 | id, |
| 1589 | title: input.title, |
| 1590 | prompt: input.prompt, |
| 1591 | enabled: input.enabled, |
| 1592 | schedule, |
| 1593 | projectId: input.projectId, |
| 1594 | threadId, |
| 1595 | threadMode, |
| 1596 | workspaceStrategy, |
| 1597 | modelSelection: input.modelSelection, |
| 1598 | runtimeMode: input.runtimeMode, |
| 1599 | interactionMode: input.interactionMode, |
| 1600 | createdBy: existingTask?.createdBy ?? input.createdBy ?? "user", |
| 1601 | creationSource: input.creationSource ?? "web", |
| 1602 | createdAt: existingTask?.createdAt ?? iso(now), |
| 1603 | updatedAt: iso(now), |
| 1604 | nextRunAt: scheduleUnchanged |
| 1605 | ? existingTask.nextRunAt |
| 1606 | : nextRunAt({ enabled: input.enabled, schedule }, now), |
| 1607 | lastRunAt: existingTask?.lastRunAt ?? null, |
| 1608 | lastRunStatus: existingTask?.lastRunStatus ?? "never", |
| 1609 | lastRunError: existingTask?.lastRunError ?? null, |
| 1610 | runCount: existingTask?.runCount ?? 0, |
| 1611 | }; |
| 1612 | yield* saveTask( |
| 1613 | task, |
| 1614 | input.requireExisting === true, |
| 1615 | scheduleUnchanged ? existingTask.schedule : null, |
| 1616 | threadId === null && input.unbindThread === true ? (existingTask?.threadId ?? null) : null, |
| 1617 | webhook, |
| 1618 | ); |
| 1619 | yield* notifyChanged; |
| 1620 | return { task: (yield* findTask(id)) ?? task }; |
| 1621 | }); |
| 1622 | |
| 1623 | const setEnabled: ScheduledTaskService["Service"]["setEnabled"] = Effect.fn( |
| 1624 | "ScheduledTaskService.setEnabled", |
| 1625 | )(function* (input) { |
| 1626 | const existing = yield* loadTask(input.id); |
| 1627 | if (existing.enabled === input.enabled) return { task: existing }; |
| 1628 | if ( |
| 1629 | input.enabled && |
| 1630 | existing.schedule.type === "webhook" && |
| 1631 | webhookPolicyBlock !== undefined |
| 1632 | ) { |
| 1633 | return yield* taskError(webhookPolicyBlock, { taskId: input.id }); |
| 1634 | } |
| 1635 | const modeDenial = |
| 1636 | input.enabled && existing.threadId === null |
| 1637 | ? managedRuntimeModeDenial(managedPolicy.summary, existing.runtimeMode) |
| 1638 | : undefined; |
| 1639 | if (modeDenial !== undefined) { |
| 1640 | return yield* taskError(modeDenial, { taskId: input.id }); |
| 1641 | } |
| 1642 | const now = yield* localNow; |
| 1643 | yield* requireFutureOnce( |
| 1644 | { enabled: input.enabled, schedule: existing.schedule }, |
| 1645 | now, |
| 1646 | input.id, |
| 1647 | ); |
| 1648 | const next = nextRunAt({ enabled: input.enabled, schedule: existing.schedule }, now); |
| 1649 | // RETURNING so a task deleted between the load and this UPDATE is a |
| 1650 | // visible not-found error, not a false success. |
| 1651 | const updated = yield* sql<{ task_id: string }>` |
| 1652 | UPDATE scheduled_tasks |
| 1653 | SET enabled = ${input.enabled ? 1 : 0}, |
| 1654 | next_run_at = ${next}, |
| 1655 | updated_at = ${iso(now)} |
| 1656 | WHERE task_id = ${input.id} |
| 1657 | RETURNING task_id |
| 1658 | `.pipe( |
| 1659 | Effect.mapError((cause) => |
| 1660 | taskError("Could not update schedule task.", { taskId: input.id, cause }), |
| 1661 | ), |
| 1662 | ); |
| 1663 | if (updated.length === 0) { |
| 1664 | return yield* taskError("Schedule task not found.", { taskId: input.id }); |
| 1665 | } |
| 1666 | yield* notifyChanged; |
| 1667 | return { |
| 1668 | task: { ...existing, enabled: input.enabled, nextRunAt: next, updatedAt: iso(now) }, |
| 1669 | }; |
| 1670 | }); |
| 1671 | |
| 1672 | const forgetWebhookTask = (id: ScheduledTaskId) => |
| 1673 | Effect.all( |
| 1674 | [ |
| 1675 | Ref.update(webhookRateWindows, (windows) => { |
| 1676 | const next = new Map(windows); |
| 1677 | next.delete(id); |
| 1678 | return next; |
| 1679 | }), |
| 1680 | Ref.update(webhookRejectedRateWindows, (windows) => { |
| 1681 | const next = new Map(windows); |
| 1682 | next.delete(id); |
| 1683 | return next; |
| 1684 | }), |
| 1685 | Ref.update(webhookDedupeLogWindows, (windows) => { |
| 1686 | const next = new Map(windows); |
| 1687 | next.delete(id); |
| 1688 | return next; |
| 1689 | }), |
| 1690 | Ref.update(webhookFilteredLogWindows, (windows) => { |
| 1691 | const next = new Map(windows); |
| 1692 | next.delete(id); |
| 1693 | return next; |
| 1694 | }), |
| 1695 | Ref.update(webhookPermits, (permits) => { |
| 1696 | const next = new Map(permits); |
| 1697 | next.delete(id); |
| 1698 | return next; |
| 1699 | }), |
| 1700 | ], |
| 1701 | { discard: true }, |
| 1702 | ); |
| 1703 | |
| 1704 | const deleteTask: ScheduledTaskService["Service"]["delete"] = (input) => |
| 1705 | deleteRow(input.id).pipe( |
| 1706 | Effect.andThen(forgetWebhookTask(input.id)), |
| 1707 | Effect.andThen(notifyChanged), |
| 1708 | Effect.as({ id: input.id }), |
| 1709 | ); |
| 1710 | |
| 1711 | const deleteForThread = Effect.fn("ScheduledTaskService.deleteForThread")(function* ( |
| 1712 | threadId: ThreadId, |
| 1713 | ) { |
| 1714 | // Check the current binding in the transaction: an automation may have |
| 1715 | // been unbound or moved since thread deletion queued this cleanup. |
| 1716 | const removed = yield* sql |
| 1717 | .withTransaction( |
| 1718 | Effect.gen(function* () { |
| 1719 | yield* sql` |
| 1720 | DELETE FROM scheduled_task_webhook_deliveries |
| 1721 | WHERE task_id IN (SELECT task_id FROM scheduled_tasks WHERE thread_id = ${threadId}) |
| 1722 | `; |
| 1723 | return yield* sql<{ task_id: ScheduledTaskId }>` |
| 1724 | DELETE FROM scheduled_tasks WHERE thread_id = ${threadId} RETURNING task_id |
| 1725 | `; |
| 1726 | }), |
| 1727 | ) |
| 1728 | .pipe( |
| 1729 | Effect.mapError((cause) => |
| 1730 | taskError("Could not delete thread's schedule tasks.", { cause }), |
| 1731 | ), |
| 1732 | ); |
| 1733 | yield* Effect.forEach(removed, ({ task_id }) => forgetWebhookTask(task_id), { |
| 1734 | discard: true, |
| 1735 | }); |
| 1736 | if (removed.length > 0) yield* notifyChanged; |
| 1737 | }); |
| 1738 | |
| 1739 | const moveThreadTasks: ScheduledTaskService["Service"]["moveThreadTasks"] = Effect.fn( |
| 1740 | "ScheduledTaskService.moveThreadTasks", |
| 1741 | )(function* (input) { |
| 1742 | if (input.threadIds.length === 0) return; |
| 1743 | const moved = yield* sql<{ task_id: ScheduledTaskId }>` |
| 1744 | UPDATE scheduled_tasks SET project_id = ${input.projectId} |
| 1745 | WHERE project_id != ${input.projectId} AND thread_id IN ${sql.in(input.threadIds)} |
| 1746 | RETURNING task_id |
| 1747 | `.pipe( |
| 1748 | Effect.mapError((cause) => |
| 1749 | taskError("Could not move the thread's schedule tasks.", { cause }), |
| 1750 | ), |
| 1751 | ); |
| 1752 | if (moved.length > 0) yield* notifyChanged; |
| 1753 | }); |
| 1754 | |
| 1755 | const runNow: ScheduledTaskService["Service"]["runNow"] = Effect.fn( |
| 1756 | "ScheduledTaskService.runNow", |
| 1757 | )(function* (input: ScheduledTaskRunNowInput) { |
| 1758 | const task = yield* loadTask(input.id); |
| 1759 | if (task.schedule.type === "webhook") { |
| 1760 | return yield* taskError("Webhook tasks run when their URL receives a request.", { |
| 1761 | taskId: input.id, |
| 1762 | }); |
| 1763 | } |
| 1764 | return yield* runTask(task, "manual").pipe( |
| 1765 | Effect.mapError((cause) => |
| 1766 | isScheduledTaskError(cause) |
| 1767 | ? cause |
| 1768 | : taskError("Could not run schedule task.", { taskId: input.id, cause }), |
| 1769 | ), |
| 1770 | ); |
| 1771 | }); |
| 1772 | |
| 1773 | const rotateWebhookToken: ScheduledTaskService["Service"]["rotateWebhookToken"] = Effect.fn( |
| 1774 | "ScheduledTaskService.rotateWebhookToken", |
| 1775 | )(function* (input) { |
| 1776 | const task = yield* loadTask(input.id); |
| 1777 | if (task.schedule.type !== "webhook") { |
| 1778 | return yield* taskError("Only webhook tasks have a URL token.", { taskId: input.id }); |
| 1779 | } |
| 1780 | const token = yield* newWebhookToken; |
| 1781 | const now = yield* localNow; |
| 1782 | // Matching created_at keeps a rotation from landing on a task deleted |
| 1783 | // and recreated under the same id since it was loaded. |
| 1784 | const updated = yield* sql<{ task_id: string }>` |
| 1785 | UPDATE scheduled_tasks |
| 1786 | SET webhook_token = ${token}, updated_at = ${iso(now)} |
| 1787 | WHERE task_id = ${input.id} AND created_at = ${task.createdAt} |
| 1788 | RETURNING task_id |
| 1789 | `.pipe( |
| 1790 | Effect.mapError((cause) => |
| 1791 | taskError("Could not rotate webhook token.", { taskId: input.id, cause }), |
| 1792 | ), |
| 1793 | ); |
| 1794 | if (updated.length === 0) { |
| 1795 | return yield* taskError("Schedule task was deleted or replaced.", { taskId: input.id }); |
| 1796 | } |
| 1797 | yield* notifyChanged; |
| 1798 | return { task: yield* loadTask(input.id) }; |
| 1799 | }); |
| 1800 | |
| 1801 | const deliveryHeaders = (row: WebhookDeliveryRow): Readonly<Record<string, string>> => |
| 1802 | Option.getOrElse(decodeHeadersJson(row.headers_json), () => ({})); |
| 1803 | const decodeDeliverySummary = (row: WebhookDeliveryRow) => ({ |
| 1804 | id: ScheduledTaskWebhookDeliveryId.make(row.delivery_id), |
| 1805 | taskId: ScheduledTaskId.make(row.task_id), |
| 1806 | receivedAt: row.received_at, |
| 1807 | method: row.method, |
| 1808 | contentType: deliveryHeaders(row)["content-type"] ?? null, |
| 1809 | bodyBytes: row.body_bytes, |
| 1810 | outcome: row.outcome as ScheduledTaskWebhookDeliveryOutcome, |
| 1811 | signatureVerified: row.signature_verified === 1, |
| 1812 | pendingWindow: row.outcome === "accepted" && row.pending_window === 1, |
| 1813 | missingFields: Option.getOrElse(decodeMissingFieldsJson(row.missing_fields_json), () => []), |
| 1814 | error: row.error, |
| 1815 | }); |
| 1816 | |
| 1817 | const listWebhookDeliveries: ScheduledTaskService["Service"]["listWebhookDeliveries"] = ( |
| 1818 | input, |
| 1819 | ) => |
| 1820 | sql<WebhookDeliveryRow>` |
| 1821 | SELECT d.*, EXISTS (SELECT 1 FROM scheduled_task_webhook_pending p |
| 1822 | WHERE p.delivery_id = d.delivery_id AND p.state IN ('pending', 'dispatching')) AS pending_window |
| 1823 | FROM scheduled_task_webhook_deliveries d WHERE task_id = ${input.id} |
| 1824 | ORDER BY received_at DESC, rowid DESC |
| 1825 | `.pipe( |
| 1826 | NodeSqliteClient.readOnly, |
| 1827 | Effect.map((rows) => ({ |
| 1828 | deliveries: rows |
| 1829 | .filter( |
| 1830 | (row) => input.acceptAnyOutcome === true || LEGACY_DELIVERY_OUTCOMES.has(row.outcome), |
| 1831 | ) |
| 1832 | .map(decodeDeliverySummary), |
| 1833 | })), |
| 1834 | Effect.mapError((cause) => |
| 1835 | taskError("Could not list webhook deliveries.", { taskId: input.id, cause }), |
| 1836 | ), |
| 1837 | ); |
| 1838 | |
| 1839 | const getWebhookDelivery: ScheduledTaskService["Service"]["getWebhookDelivery"] = Effect.fn( |
| 1840 | "ScheduledTaskService.getWebhookDelivery", |
| 1841 | )(function* (input) { |
| 1842 | const rows = yield* sql<WebhookDeliveryRow>` |
| 1843 | SELECT d.*, EXISTS (SELECT 1 FROM scheduled_task_webhook_pending p |
| 1844 | WHERE p.delivery_id = d.delivery_id AND p.state IN ('pending', 'dispatching')) AS pending_window |
| 1845 | FROM scheduled_task_webhook_deliveries d WHERE task_id = ${input.id} AND delivery_id = ${input.deliveryId} |
| 1846 | `.pipe( |
| 1847 | NodeSqliteClient.readOnly, |
| 1848 | Effect.mapError((cause) => |
| 1849 | taskError("Could not load webhook delivery.", { taskId: input.id, cause }), |
| 1850 | ), |
| 1851 | ); |
| 1852 | const row = rows[0]; |
| 1853 | if ( |
| 1854 | row === undefined || |
| 1855 | (input.acceptAnyOutcome !== true && !LEGACY_DELIVERY_OUTCOMES.has(row.outcome)) |
| 1856 | ) { |
| 1857 | return yield* taskError("Webhook delivery not found.", { taskId: input.id }); |
| 1858 | } |
| 1859 | return { |
| 1860 | delivery: { |
| 1861 | ...decodeDeliverySummary(row), |
| 1862 | query: row.query, |
| 1863 | headers: deliveryHeaders(row), |
| 1864 | body: row.body, |
| 1865 | bodyTruncated: row.body_truncated === 1, |
| 1866 | renderedPrompt: row.rendered_prompt, |
| 1867 | }, |
| 1868 | }; |
| 1869 | }); |
| 1870 | |
| 1871 | const deliveryRequestBody = (request: WebhookTriggerRequest) => { |
| 1872 | const truncated = request.body.byteLength > WEBHOOK_DELIVERY_LOG_BODY_LIMIT; |
| 1873 | return { |
| 1874 | truncated, |
| 1875 | // Streaming decode drops a character the byte limit split, instead of ending in U+FFFD. |
| 1876 | body: truncated |
| 1877 | ? new TextDecoder().decode(request.body.subarray(0, WEBHOOK_DELIVERY_LOG_BODY_LIMIT), { |
| 1878 | stream: true, |
| 1879 | }) |
| 1880 | : request.bodyText, |
| 1881 | }; |
| 1882 | }; |
| 1883 | const replaceDeliveryRequest = (deliveryId: string, request: WebhookTriggerRequest) => { |
| 1884 | const logged = deliveryRequestBody(request); |
| 1885 | return sql`UPDATE scheduled_task_webhook_deliveries SET |
| 1886 | method = ${request.method}, query = ${redactQuery(request.query)}, |
| 1887 | headers_json = ${encodeHeadersJson(redactHeaders(request.headers))}, body = ${logged.body}, |
| 1888 | body_bytes = ${request.body.byteLength}, body_truncated = ${logged.truncated ? 1 : 0} |
| 1889 | WHERE delivery_id = ${deliveryId}`.pipe(Effect.asVoid, Effect.mapError(dedupeError)); |
| 1890 | }; |
| 1891 | |
| 1892 | // Each suppression bucket retains its own rows, never real run history. |
| 1893 | const deliveryBucket = (outcome: string, error: string | undefined) => |
| 1894 | outcome === "filtered" |
| 1895 | ? "filtered" |
| 1896 | : outcome === "duplicate" || |
| 1897 | outcome === "coalesced" || |
| 1898 | outcome === "rate_limited" || |
| 1899 | error === PROMPT_TOO_LONG |
| 1900 | ? "dedupe" |
| 1901 | : "main"; |
| 1902 | const pruneDeliveries = ( |
| 1903 | taskId: string, |
| 1904 | retentionBucket: ReturnType<typeof deliveryBucket>, |
| 1905 | ) => { |
| 1906 | const rowBucket = sql.literal(`CASE WHEN outcome = 'filtered' THEN 'filtered' |
| 1907 | WHEN outcome IN ('duplicate', 'coalesced', 'rate_limited') OR error = 'The filled-in prompt is too long.' THEN 'dedupe' |
| 1908 | ELSE 'main' END`); |
| 1909 | return sql` |
| 1910 | DELETE FROM scheduled_task_webhook_deliveries |
| 1911 | WHERE task_id = ${taskId} AND ${rowBucket} = ${retentionBucket} |
| 1912 | AND delivery_id NOT IN ( |
| 1913 | SELECT delivery_id FROM scheduled_task_webhook_deliveries |
| 1914 | WHERE task_id = ${taskId} AND ${rowBucket} = ${retentionBucket} |
| 1915 | ORDER BY received_at DESC, rowid DESC |
| 1916 | LIMIT ${retentionBucket === "main" ? WEBHOOK_DELIVERY_RETENTION : retentionBucket === "filtered" ? WEBHOOK_FILTERED_DELIVERY_RETENTION : WEBHOOK_DEDUPE_DELIVERY_RETENTION} |
| 1917 | ) |
| 1918 | `; |
| 1919 | }; |
| 1920 | |
| 1921 | const recordDelivery = (input: { |
| 1922 | readonly id: string; |
| 1923 | readonly taskId: ScheduledTaskId; |
| 1924 | readonly receivedAt: string; |
| 1925 | readonly request: WebhookTriggerRequest; |
| 1926 | readonly outcome: ScheduledTaskWebhookDeliveryOutcome; |
| 1927 | readonly signatureVerified: boolean; |
| 1928 | readonly missing: ReadonlyArray<string>; |
| 1929 | readonly renderedPrompt: string | null; |
| 1930 | readonly error?: string; |
| 1931 | }) => { |
| 1932 | const { truncated, body: loggedBody } = deliveryRequestBody(input.request); |
| 1933 | return sql |
| 1934 | .withTransaction( |
| 1935 | Effect.gen(function* () { |
| 1936 | // Require the task to exist so deletes cannot leave deliveries for a recreated task. |
| 1937 | yield* sql` |
| 1938 | INSERT INTO scheduled_task_webhook_deliveries ( |
| 1939 | delivery_id, task_id, received_at, method, query, headers_json, body, |
| 1940 | body_bytes, body_truncated, outcome, signature_verified, |
| 1941 | missing_fields_json, rendered_prompt, error |
| 1942 | ) |
| 1943 | SELECT |
| 1944 | ${input.id}, ${input.taskId}, ${input.receivedAt}, ${input.request.method}, |
| 1945 | ${redactQuery(input.request.query)}, |
| 1946 | ${encodeHeadersJson(redactHeaders(input.request.headers))}, |
| 1947 | ${loggedBody}, |
| 1948 | ${input.request.body.byteLength}, ${truncated ? 1 : 0}, ${input.outcome}, |
| 1949 | ${input.signatureVerified ? 1 : 0}, ${encodeMissingFieldsJson(input.missing)}, |
| 1950 | ${input.renderedPrompt?.slice(0, WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT) ?? null}, |
| 1951 | ${input.error ?? null} |
| 1952 | WHERE EXISTS (SELECT 1 FROM scheduled_tasks WHERE task_id = ${input.taskId}) |
| 1953 | -- A held request retried after a rate limit reuses its relay |
| 1954 | -- delivery id; the newer attempt replaces the logged one. |
| 1955 | ON CONFLICT (delivery_id) DO UPDATE SET |
| 1956 | outcome = excluded.outcome, |
| 1957 | signature_verified = excluded.signature_verified, |
| 1958 | missing_fields_json = excluded.missing_fields_json, |
| 1959 | rendered_prompt = excluded.rendered_prompt, |
| 1960 | error = excluded.error |
| 1961 | `; |
| 1962 | yield* pruneDeliveries(input.taskId, deliveryBucket(input.outcome, input.error)); |
| 1963 | }), |
| 1964 | ) |
| 1965 | .pipe( |
| 1966 | Effect.mapError((cause) => |
| 1967 | taskError("Could not record webhook delivery.", { taskId: input.taskId, cause }), |
| 1968 | ), |
| 1969 | ); |
| 1970 | }; |
| 1971 | |
| 1972 | const markDeliveryFailed = (deliveryId: string, message: string) => |
| 1973 | sql |
| 1974 | .withTransaction( |
| 1975 | Effect.gen(function* () { |
| 1976 | const [updated] = yield* sql<{ task_id: string }>` |
| 1977 | UPDATE scheduled_task_webhook_deliveries |
| 1978 | SET outcome = 'dispatch_failed', error = ${message} |
| 1979 | WHERE delivery_id = ${deliveryId} |
| 1980 | RETURNING task_id |
| 1981 | `; |
| 1982 | // An oversized prompt moves the row into the suppression bucket. |
| 1983 | if (updated !== undefined) |
| 1984 | yield* pruneDeliveries(updated.task_id, deliveryBucket("dispatch_failed", message)); |
| 1985 | }), |
| 1986 | ) |
| 1987 | .pipe(Effect.ignore); |
| 1988 | |
| 1989 | /** Sliding one-minute window for deliveries that can start work. */ |
| 1990 | const takeRateSlot = ( |
| 1991 | id: ScheduledTaskId, |
| 1992 | nowMs: number, |
| 1993 | rateWindows = webhookRateWindows, |
| 1994 | reservation = { at: nowMs }, |
| 1995 | ) => |
| 1996 | Ref.modify( |
| 1997 | rateWindows, |
| 1998 | ( |
| 1999 | windows, |
| 2000 | ): readonly [ |
| 2001 | "allowed" | "first_rejected" | "rejected", |
| 2002 | ReadonlyMap<ScheduledTaskId, RateWindow>, |
| 2003 | ] => { |
| 2004 | const current = windows.get(id); |
| 2005 | const recent = (current?.accepted ?? []).filter((entry) => nowMs - entry.at < 60_000); |
| 2006 | if (recent.length >= WEBHOOK_RATE_LIMIT_PER_MINUTE) { |
| 2007 | const first = current?.rejectedLogged !== true; |
| 2008 | return [ |
| 2009 | first ? "first_rejected" : "rejected", |
| 2010 | new Map(windows).set(id, { accepted: recent, rejectedLogged: true }), |
| 2011 | ]; |
| 2012 | } |
| 2013 | return [ |
| 2014 | "allowed", |
| 2015 | new Map(windows).set(id, { accepted: [...recent, reservation], rejectedLogged: false }), |
| 2016 | ]; |
| 2017 | }, |
| 2018 | ); |
| 2019 | |
| 2020 | const takeDedupeLogSlot = (id: ScheduledTaskId, nowMs: number) => |
| 2021 | Ref.modify(webhookDedupeLogWindows, (windows) => { |
| 2022 | const recent = (windows.get(id) ?? []).filter((at) => nowMs - at < 60_000); |
| 2023 | return recent.length >= WEBHOOK_DEDUPE_LOG_PER_MINUTE |
| 2024 | ? ([false, windows] as const) |
| 2025 | : ([true, new Map(windows).set(id, [...recent, nowMs])] as const); |
| 2026 | }); |
| 2027 | const releaseReplay = (deliveryId: string) => |
| 2028 | releaseWebhookReplay(deliveryId).pipe( |
| 2029 | Effect.provideService(SqlClient.SqlClient, sql), |
| 2030 | Effect.mapError(dedupeError), |
| 2031 | ); |
| 2032 | /** Sliding one-minute window of filtered requests written to the delivery log. */ |
| 2033 | const takeFilteredLogSlot = (id: ScheduledTaskId, nowMs: number) => |
| 2034 | Ref.modify( |
| 2035 | webhookFilteredLogWindows, |
| 2036 | (windows): readonly [boolean, ReadonlyMap<ScheduledTaskId, ReadonlyArray<number>>] => { |
| 2037 | const recent = (windows.get(id) ?? []).filter((at) => nowMs - at < 60_000); |
| 2038 | return recent.length >= WEBHOOK_FILTERED_LOG_PER_MINUTE |
| 2039 | ? [false, windows] |
| 2040 | : [true, new Map(windows).set(id, [...recent, nowMs])]; |
| 2041 | }, |
| 2042 | ); |
| 2043 | |
| 2044 | const webhookPermit = Effect.fn("ScheduledTaskService.webhookPermit")(function* ( |
| 2045 | id: ScheduledTaskId, |
| 2046 | ) { |
| 2047 | const existing = (yield* Ref.get(webhookPermits)).get(id); |
| 2048 | if (existing !== undefined) return existing; |
| 2049 | const created = yield* Semaphore.make(1); |
| 2050 | return yield* Ref.modify(webhookPermits, (permits) => { |
| 2051 | const raced = permits.get(id); |
| 2052 | return raced === undefined |
| 2053 | ? [created, new Map(permits).set(id, created)] |
| 2054 | : [raced, permits]; |
| 2055 | }); |
| 2056 | }); |
| 2057 | |
| 2058 | const releaseFailedDelivery = Effect.fn("ScheduledTaskService.releaseFailedDelivery")( |
| 2059 | function* (deliveryId: string) { |
| 2060 | yield* releaseReplay(deliveryId); |
| 2061 | yield* sql`DELETE FROM scheduled_task_webhook_relay_deliveries WHERE relay_delivery_id IN ( |
| 2062 | SELECT substr(value, ${"delivery:relay:".length + 1}) FROM scheduled_task_webhook_pending, json_each(delivery_ids_json) |
| 2063 | WHERE delivery_id = ${deliveryId} AND value LIKE 'delivery:relay:%' |
| 2064 | )`.pipe(Effect.mapError(dedupeError)); |
| 2065 | if (deliveryId.startsWith("delivery:relay:")) |
| 2066 | yield* sql`DELETE FROM scheduled_task_webhook_relay_deliveries WHERE relay_delivery_id = ${deliveryId.slice("delivery:relay:".length)}`.pipe( |
| 2067 | Effect.mapError(dedupeError), |
| 2068 | ); |
| 2069 | }, |
| 2070 | (effect) => sql.withTransaction(effect).pipe(Effect.mapError(dedupeError)), |
| 2071 | ); |
| 2072 | |
| 2073 | const runWebhookDelivery = Effect.fn("ScheduledTaskService.runWebhookDelivery")(function* ( |
| 2074 | task: ScheduledTask, |
| 2075 | deliveryId: string, |
| 2076 | prompt: string, |
| 2077 | ) { |
| 2078 | const runOutcome = (outcome: "started" | "skipped" | "failed") => |
| 2079 | Effect.all([ |
| 2080 | Effect.annotateCurrentSpan({ "scheduled_task.webhook.run_outcome": outcome }), |
| 2081 | Metrics.increment(Metrics.webhookRunsTotal, { outcome }), |
| 2082 | ]); |
| 2083 | const dispatched = yield* Ref.make(false); |
| 2084 | return yield* runTask(task, "webhook", { deliveryId, prompt, dispatched }).pipe( |
| 2085 | Effect.flatMap(({ dispatchFailed }) => |
| 2086 | dispatchFailed |
| 2087 | ? runOutcome("failed").pipe( |
| 2088 | Effect.andThen(markDeliveryFailed(deliveryId, "The run failed to start.")), |
| 2089 | Effect.andThen(releaseFailedDelivery(deliveryId)), |
| 2090 | ) |
| 2091 | : runOutcome("started"), |
| 2092 | ), |
| 2093 | Effect.catchTags({ |
| 2094 | WebhookDeliverySkipped: (skipped) => |
| 2095 | runOutcome("skipped").pipe( |
| 2096 | Effect.andThen(markDeliveryFailed(deliveryId, skipped.reason)), |
| 2097 | Effect.andThen(releaseFailedDelivery(deliveryId)), |
| 2098 | ), |
| 2099 | }), |
| 2100 | Effect.as(true), |
| 2101 | // Dispatch success owns the replay key even if later bookkeeping fails. |
| 2102 | Effect.catchCause((cause) => |
| 2103 | Effect.gen(function* () { |
| 2104 | const started = yield* Ref.get(dispatched); |
| 2105 | yield* Effect.logWarning("Webhook dispatch failed", { taskId: task.id, cause }); |
| 2106 | yield* runOutcome(started ? "started" : "failed"); |
| 2107 | yield* markDeliveryFailed( |
| 2108 | deliveryId, |
| 2109 | started |
| 2110 | ? "The run started, but completion could not be recorded." |
| 2111 | : "The run failed to start.", |
| 2112 | ); |
| 2113 | if (!started) { |
| 2114 | yield* sql |
| 2115 | .withTransaction( |
| 2116 | Effect.gen(function* () { |
| 2117 | const counted = |
| 2118 | yield* sql`UPDATE scheduled_task_webhook_pending SET state = 'failed' |
| 2119 | WHERE delivery_id = ${deliveryId} AND state = 'running' RETURNING delivery_id`; |
| 2120 | if (counted.length > 0) |
| 2121 | yield* sql`UPDATE scheduled_tasks SET run_count = run_count + 1 |
| 2122 | WHERE task_id = ${task.id} AND created_at = ${task.createdAt}`; |
| 2123 | }), |
| 2124 | ) |
| 2125 | .pipe(Effect.mapError(dedupeError)); |
| 2126 | yield* releaseFailedDelivery(deliveryId); |
| 2127 | } |
| 2128 | // Pending windows retry the same command id to finish bookkeeping. |
| 2129 | return !started; |
| 2130 | }), |
| 2131 | ), |
| 2132 | ); |
| 2133 | }); |
| 2134 | |
| 2135 | // Delivery attributes belong to the enclosing triggerWebhook span. |
| 2136 | const observeDelivery = Effect.fnUntraced(function* ( |
| 2137 | request: WebhookTriggerRequest, |
| 2138 | outcome: WebhookDeliveryOutcome, |
| 2139 | receivedAt: DateTime.DateTime | undefined, |
| 2140 | now: DateTime.DateTime | undefined, |
| 2141 | ) { |
| 2142 | const source = request.relayDeliveryId === undefined ? "direct" : "relay"; |
| 2143 | yield* Effect.annotateCurrentSpan({ |
| 2144 | "scheduled_task.webhook.outcome": outcome, |
| 2145 | "scheduled_task.webhook.source": source, |
| 2146 | }); |
| 2147 | yield* Metrics.increment(Metrics.webhookDeliveriesTotal, { outcome, source }); |
| 2148 | if (request.receivedAt !== undefined && receivedAt !== undefined && now !== undefined) { |
| 2149 | const heldMs = Math.max( |
| 2150 | 0, |
| 2151 | DateTime.toEpochMillis(now) - DateTime.toEpochMillis(receivedAt), |
| 2152 | ); |
| 2153 | yield* Effect.annotateCurrentSpan({ "scheduled_task.webhook.held_ms": heldMs }); |
| 2154 | yield* Metric.update(Metrics.webhookHeldDelay, Duration.millis(heldMs)); |
| 2155 | } |
| 2156 | }); |
| 2157 | |
| 2158 | const triggerWebhook: ScheduledTaskService["Service"]["triggerWebhook"] = (request) => |
| 2159 | triggerWebhookUnobserved(request).pipe( |
| 2160 | // Accepted deliveries may have queued a coalescing window. |
| 2161 | Effect.ensuring(pendingWebhooks.invalidate), |
| 2162 | Effect.tapError(() => observeDelivery(request, "error", undefined, undefined)), |
| 2163 | Metrics.withMetrics({ timer: Metrics.webhookDeliveryDuration }), |
| 2164 | Effect.withSpan("ScheduledTaskService.triggerWebhook", { |
| 2165 | attributes: { |
| 2166 | "scheduled_task.webhook.method": request.method, |
| 2167 | "scheduled_task.webhook.body_bytes": request.body.byteLength, |
| 2168 | "scheduled_task.webhook.source": |
| 2169 | request.relayDeliveryId === undefined ? "direct" : "relay", |
| 2170 | }, |
| 2171 | }), |
| 2172 | ); |
| 2173 | |
| 2174 | // triggerWebhook owns the operation span and its task-id annotation. |
| 2175 | const triggerWebhookUnobserved = Effect.fnUntraced(function* (request: WebhookTriggerRequest) { |
| 2176 | // Refused before any lookup or log, so a blocked hook neither runs nor records a delivery. |
| 2177 | if (webhookPolicyBlock !== undefined) { |
| 2178 | return yield* observeDelivery(request, "blocked_by_policy", undefined, undefined).pipe( |
| 2179 | Effect.as({ _tag: "blocked_by_policy" as const }), |
| 2180 | ); |
| 2181 | } |
| 2182 | const taskId = decodeTaskId(request.hookId); |
| 2183 | const notFound = observeDelivery(request, "not_found", undefined, undefined).pipe( |
| 2184 | Effect.as({ _tag: "not_found" as const }), |
| 2185 | ); |
| 2186 | if (Option.isNone(taskId)) return yield* notFound; |
| 2187 | const rows = yield* getRows(taskId.value).pipe( |
| 2188 | Effect.mapError((cause) => |
| 2189 | taskError("Could not load schedule task.", { taskId: taskId.value, cause }), |
| 2190 | ), |
| 2191 | ); |
| 2192 | const row = rows[0]; |
| 2193 | // A wrong token is indistinguishable from an unknown hook, so the URL |
| 2194 | // does not reveal which hooks exist. |
| 2195 | if ( |
| 2196 | row === undefined || |
| 2197 | row.webhook_token === null || |
| 2198 | !(yield* constantTimeEquals(request.token, row.webhook_token).pipe( |
| 2199 | Effect.provideService(Digest.Digest, digest), |
| 2200 | )) |
| 2201 | ) { |
| 2202 | return yield* notFound; |
| 2203 | } |
| 2204 | const task = yield* decodeRow(row); |
| 2205 | yield* Effect.annotateCurrentSpan({ "scheduled_task.id": task.id }); |
| 2206 | const schedule = task.schedule; |
| 2207 | if (schedule.type !== "webhook") return yield* notFound; |
| 2208 | |
| 2209 | const now = yield* localNow; |
| 2210 | // Anyone can reach the tunnel directly and set the relay's header, so |
| 2211 | // a receive time is never later than now: a future one would pin the |
| 2212 | // delivery in the log and slip past the task's max age. |
| 2213 | const receivedAt = |
| 2214 | request.receivedAt === undefined |
| 2215 | ? now |
| 2216 | : DateTime.min( |
| 2217 | Option.getOrElse(DateTime.make(request.receivedAt), () => now), |
| 2218 | now, |
| 2219 | ); |
| 2220 | const observe = (outcome: WebhookDeliveryOutcome) => |
| 2221 | observeDelivery(request, outcome, receivedAt, now); |
| 2222 | const deliveryId = ScheduledTaskWebhookDeliveryId.make( |
| 2223 | request.relayDeliveryId === undefined |
| 2224 | ? `delivery:${yield* random |
| 2225 | .uuidV4() |
| 2226 | .pipe( |
| 2227 | Effect.mapError((cause) => taskError("Could not generate delivery id.", { cause })), |
| 2228 | )}` |
| 2229 | : `delivery:relay:${request.relayDeliveryId}`, |
| 2230 | ); |
| 2231 | // Claimed in its own table, kept longer than the relay holds a request, |
| 2232 | // because the delivery log is trimmed. A rate-limited delivery stays |
| 2233 | // held on the relay, so it must give up its claim or the next pass |
| 2234 | // would treat it as already delivered. |
| 2235 | let claimAcquired = false; |
| 2236 | const releaseClaim = <A>(result: A) => |
| 2237 | !claimAcquired || request.relayDeliveryId === undefined |
| 2238 | ? Effect.succeed(result) |
| 2239 | : sql` |
| 2240 | DELETE FROM scheduled_task_webhook_relay_deliveries |
| 2241 | WHERE relay_delivery_id = ${request.relayDeliveryId} |
| 2242 | `.pipe(Effect.ignore, Effect.as(result)); |
| 2243 | // From the claim until the run is forked nothing may interrupt: a |
| 2244 | // request dropped in between (the relay hangs up after its timeout) |
| 2245 | // would leave a claimed delivery that never runs, or a queue slot that |
| 2246 | // is never released. |
| 2247 | return yield* Effect.uninterruptible( |
| 2248 | Effect.gen(function* () { |
| 2249 | if (request.relayDeliveryId !== undefined) { |
| 2250 | const claimed = yield* sql<{ relay_delivery_id: string }>` |
| 2251 | INSERT INTO scheduled_task_webhook_relay_deliveries |
| 2252 | (relay_delivery_id, task_id, seen_at) |
| 2253 | VALUES (${request.relayDeliveryId}, ${task.id}, ${iso(now)}) |
| 2254 | ON CONFLICT (relay_delivery_id) DO NOTHING |
| 2255 | RETURNING relay_delivery_id |
| 2256 | `.pipe( |
| 2257 | Effect.mapError((cause) => |
| 2258 | taskError("Could not record webhook delivery.", { taskId: task.id, cause }), |
| 2259 | ), |
| 2260 | ); |
| 2261 | if (claimed.length === 0) { |
| 2262 | yield* observe("duplicate"); |
| 2263 | return { _tag: "accepted" as const, deliveryId, outcome: "duplicate" as const }; |
| 2264 | } |
| 2265 | claimAcquired = true; |
| 2266 | yield* sql` |
| 2267 | DELETE FROM scheduled_task_webhook_relay_deliveries |
| 2268 | WHERE seen_at < ${iso(DateTime.subtract(now, { hours: 48 }))} |
| 2269 | `.pipe(Effect.ignore); |
| 2270 | } |
| 2271 | const log = ( |
| 2272 | outcome: ScheduledTaskWebhookDeliveryOutcome, |
| 2273 | details: { |
| 2274 | readonly signatureVerified?: boolean; |
| 2275 | readonly missing?: ReadonlyArray<string>; |
| 2276 | readonly renderedPrompt?: string; |
| 2277 | readonly error?: string; |
| 2278 | } = {}, |
| 2279 | ) => |
| 2280 | recordDelivery({ |
| 2281 | id: deliveryId, |
| 2282 | taskId: task.id, |
| 2283 | receivedAt: iso(receivedAt), |
| 2284 | request, |
| 2285 | outcome, |
| 2286 | signatureVerified: details.signatureVerified ?? false, |
| 2287 | missing: details.missing ?? [], |
| 2288 | renderedPrompt: details.renderedPrompt ?? null, |
| 2289 | ...(details.error === undefined ? {} : { error: details.error }), |
| 2290 | }); |
| 2291 | |
| 2292 | // Reject floods before signature work or rejection-log writes; valid |
| 2293 | // deliveries give this reservation back once validation succeeds. |
| 2294 | const rejectionReservation = { at: DateTime.toEpochMillis(now) }; |
| 2295 | const rejectionSlot = yield* takeRateSlot( |
| 2296 | task.id, |
| 2297 | DateTime.toEpochMillis(now), |
| 2298 | webhookRejectedRateWindows, |
| 2299 | rejectionReservation, |
| 2300 | ); |
| 2301 | if (rejectionSlot !== "allowed") { |
| 2302 | if (rejectionSlot === "first_rejected") yield* log("rate_limited"); |
| 2303 | yield* observe("rate_limited"); |
| 2304 | return yield* releaseClaim({ _tag: "rate_limited", outcome: "rate_limited" } as const); |
| 2305 | } |
| 2306 | if (!task.enabled) { |
| 2307 | yield* log("disabled"); |
| 2308 | yield* observe("disabled"); |
| 2309 | return { _tag: "disabled" as const }; |
| 2310 | } |
| 2311 | const signature = schedule.signature; |
| 2312 | if (signature !== null) { |
| 2313 | const verified = |
| 2314 | row.webhook_secret !== null && |
| 2315 | (yield* verifyWebhookSignature({ |
| 2316 | signature, |
| 2317 | secret: row.webhook_secret, |
| 2318 | headers: request.headers, |
| 2319 | body: request.body, |
| 2320 | }).pipe(Effect.provideService(Mac.Mac, mac))); |
| 2321 | if (!verified) { |
| 2322 | yield* log("rejected_signature"); |
| 2323 | yield* observe("rejected_signature"); |
| 2324 | return { _tag: "rejected_signature" as const }; |
| 2325 | } |
| 2326 | } |
| 2327 | const maxAgeMinutes = schedule.maxDeliveryAgeMinutes ?? null; |
| 2328 | if ( |
| 2329 | maxAgeMinutes !== null && |
| 2330 | DateTime.toEpochMillis(now) - DateTime.toEpochMillis(receivedAt) > |
| 2331 | maxAgeMinutes * 60_000 |
| 2332 | ) { |
| 2333 | yield* log("expired", { signatureVerified: signature !== null }); |
| 2334 | yield* observe("expired"); |
| 2335 | return { _tag: "expired" as const }; |
| 2336 | } |
| 2337 | |
| 2338 | const refundRejectionSlot = Ref.update(webhookRejectedRateWindows, (windows) => { |
| 2339 | const current = windows.get(task.id); |
| 2340 | if (current === undefined) return windows; |
| 2341 | return new Map(windows).set(task.id, { |
| 2342 | ...current, |
| 2343 | accepted: current.accepted.filter((entry) => entry !== rejectionReservation), |
| 2344 | }); |
| 2345 | }); |
| 2346 | |
| 2347 | const failedFilter = findFailedWebhookFilter(schedule.filters ?? [], request); |
| 2348 | if (failedFilter !== null) { |
| 2349 | if (yield* takeFilteredLogSlot(task.id, DateTime.toEpochMillis(now))) { |
| 2350 | yield* log("filtered", { |
| 2351 | signatureVerified: signature !== null, |
| 2352 | error: describeFailedWebhookFilter(failedFilter), |
| 2353 | }); |
| 2354 | } |
| 2355 | yield* refundRejectionSlot; |
| 2356 | yield* observe("filtered"); |
| 2357 | return yield* releaseClaim({ |
| 2358 | _tag: "accepted" as const, |
| 2359 | deliveryId, |
| 2360 | outcome: "filtered" as const, |
| 2361 | }); |
| 2362 | } |
| 2363 | |
| 2364 | const runWindow = (yield* Ref.get(webhookRateWindows)).get(task.id); |
| 2365 | if ( |
| 2366 | schedule.dedupe === undefined && |
| 2367 | (runWindow?.accepted.filter((entry) => DateTime.toEpochMillis(now) - entry.at < 60_000) |
| 2368 | .length ?? 0) >= WEBHOOK_RATE_LIMIT_PER_MINUTE |
| 2369 | ) { |
| 2370 | if ((yield* takeRateSlot(task.id, DateTime.toEpochMillis(now))) === "first_rejected") |
| 2371 | yield* log("rate_limited"); |
| 2372 | yield* observe("rate_limited"); |
| 2373 | return yield* releaseClaim({ _tag: "rate_limited", outcome: "rate_limited" } as const); |
| 2374 | } |
| 2375 | const reserve = takeRateSlot(task.id, DateTime.toEpochMillis(now)).pipe( |
| 2376 | Effect.flatMap((slot) => (slot === "allowed" ? Effect.void : Effect.fail(slot))), |
| 2377 | ); |
| 2378 | const queueKey = `${task.id}\u0000${task.createdAt}`; |
| 2379 | const checkCapacity = Effect.gen(function* () { |
| 2380 | const [row] = yield* sql<{ |
| 2381 | count: number; |
| 2382 | }>`SELECT COUNT(*) AS count FROM scheduled_task_webhook_pending |
| 2383 | WHERE task_id = ${task.id} AND state IN ('pending', 'dispatching', 'running')`; |
| 2384 | if ( |
| 2385 | (row?.count ?? 0) + ((yield* Ref.get(webhookQueued)).get(queueKey) ?? 0) >= |
| 2386 | WEBHOOK_MAX_QUEUED_PER_TASK |
| 2387 | ) |
| 2388 | return yield* Effect.fail("queue_full" as const); |
| 2389 | }).pipe( |
| 2390 | Effect.mapError((cause) => (cause === "queue_full" ? cause : dedupeError(cause))), |
| 2391 | ); |
| 2392 | const admit = <A, E>(effect: Effect.Effect<A, E>) => |
| 2393 | sql.withTransaction(effect).pipe( |
| 2394 | Effect.catch((cause) => |
| 2395 | cause === "first_rejected" || cause === "rejected" || cause === "queue_full" |
| 2396 | ? Effect.succeed({ |
| 2397 | outcome: |
| 2398 | cause === "queue_full" |
| 2399 | ? ("queue_full" as const) |
| 2400 | : ("rate_limited" as const), |
| 2401 | log: cause === "first_rejected", |
| 2402 | }) |
| 2403 | : Effect.fail(isScheduledTaskError(cause) ? cause : dedupeError(cause)), |
| 2404 | ), |
| 2405 | ); |
| 2406 | const deduped = |
| 2407 | schedule.dedupe === undefined |
| 2408 | ? "run" |
| 2409 | : yield* admit( |
| 2410 | Effect.gen(function* () { |
| 2411 | const outcome = yield* acceptWebhookDedupe( |
| 2412 | task, |
| 2413 | request, |
| 2414 | deliveryId, |
| 2415 | DateTime.toEpochMillis(now), |
| 2416 | reserve, |
| 2417 | takeDedupeLogSlot(task.id, DateTime.toEpochMillis(now)), |
| 2418 | checkCapacity, |
| 2419 | (ownerId) => replaceDeliveryRequest(ownerId, request), |
| 2420 | ).pipe(Effect.provideService(SqlClient.SqlClient, sql)); |
| 2421 | if ( |
| 2422 | outcome === "pending" || |
| 2423 | outcome === "coalesced" || |
| 2424 | (outcome === "duplicate" && |
| 2425 | (yield* takeDedupeLogSlot(task.id, DateTime.toEpochMillis(now)))) |
| 2426 | ) |
| 2427 | yield* log(outcome === "pending" ? "accepted" : outcome, { |
| 2428 | signatureVerified: signature !== null, |
| 2429 | }); |
| 2430 | return outcome; |
| 2431 | }), |
| 2432 | ); |
| 2433 | if (typeof deduped !== "string") { |
| 2434 | yield* refundRejectionSlot; |
| 2435 | if (deduped.log) yield* log("rate_limited"); |
| 2436 | yield* observe(deduped.outcome); |
| 2437 | return yield* releaseClaim({ _tag: "rate_limited", outcome: deduped.outcome } as const); |
| 2438 | } |
| 2439 | if (deduped !== "run") { |
| 2440 | if (deduped !== "coalesced" && deduped !== "coalesced_unlogged") |
| 2441 | yield* refundRejectionSlot; |
| 2442 | const outcome = |
| 2443 | deduped === "pending" |
| 2444 | ? "accepted" |
| 2445 | : deduped === "coalesced_unlogged" |
| 2446 | ? "coalesced" |
| 2447 | : deduped; |
| 2448 | yield* observe(outcome); |
| 2449 | if (deduped === "duplicate") |
| 2450 | return yield* releaseClaim({ _tag: "accepted", deliveryId, outcome } as const); |
| 2451 | claimAcquired = false; |
| 2452 | return { _tag: "accepted", deliveryId, outcome } as const; |
| 2453 | } |
| 2454 | const frameId = yield* random.randomBytes(6).pipe( |
| 2455 | Effect.map((bytes) => Buffer.from(bytes).toString("hex")), |
| 2456 | Effect.mapError((cause) => taskError("Could not generate frame id.", { cause })), |
| 2457 | ); |
| 2458 | const rendered = renderWebhookPrompt(task.prompt, request, frameId); |
| 2459 | // Providers trim the prompt before checking, so whitespace around |
| 2460 | // it is free. |
| 2461 | if (rendered.prompt.trim().length > PROVIDER_SEND_TURN_MAX_INPUT_CHARS) { |
| 2462 | yield* log("dispatch_failed", { |
| 2463 | signatureVerified: signature !== null, |
| 2464 | missing: rendered.missing, |
| 2465 | renderedPrompt: rendered.prompt, |
| 2466 | error: PROMPT_TOO_LONG, |
| 2467 | }); |
| 2468 | yield* releaseReplay(deliveryId); |
| 2469 | yield* observe("prompt_too_long"); |
| 2470 | return yield* releaseClaim({ |
| 2471 | _tag: "accepted" as const, |
| 2472 | deliveryId, |
| 2473 | outcome: "prompt_too_long" as const, |
| 2474 | }); |
| 2475 | } |
| 2476 | // Bound the deliveries one task holds, so steady traffic to a stuck |
| 2477 | // task cannot pile up parked fibers. |
| 2478 | yield* refundRejectionSlot; |
| 2479 | let queueReserved = false; |
| 2480 | const release = Effect.suspend(() => { |
| 2481 | if (!queueReserved) return Effect.void; |
| 2482 | queueReserved = false; |
| 2483 | return Ref.update(webhookQueued, (counts) => { |
| 2484 | const next = new Map(counts); |
| 2485 | const count = (next.get(queueKey) ?? 1) - 1; |
| 2486 | if (count <= 0) next.delete(queueKey); |
| 2487 | else next.set(queueKey, count); |
| 2488 | return next; |
| 2489 | }); |
| 2490 | }); |
| 2491 | // Serialize immediate reservations with persisted window admission. |
| 2492 | const reserveImmediate = Effect.gen(function* () { |
| 2493 | yield* checkCapacity; |
| 2494 | yield* reserve; |
| 2495 | yield* Ref.update(webhookQueued, (counts) => |
| 2496 | new Map(counts).set(queueKey, (counts.get(queueKey) ?? 0) + 1), |
| 2497 | ); |
| 2498 | queueReserved = true; |
| 2499 | }); |
| 2500 | const admitted = yield* admit( |
| 2501 | Effect.gen(function* () { |
| 2502 | const outcome = yield* acceptImmediateWebhookDedupe( |
| 2503 | task, |
| 2504 | request, |
| 2505 | deliveryId, |
| 2506 | DateTime.toEpochMillis(now), |
| 2507 | reserveImmediate, |
| 2508 | ).pipe(Effect.provideService(SqlClient.SqlClient, sql)); |
| 2509 | if (outcome === "duplicate") { |
| 2510 | if (yield* takeDedupeLogSlot(task.id, DateTime.toEpochMillis(now))) |
| 2511 | yield* log("duplicate", { signatureVerified: signature !== null }); |
| 2512 | } else |
| 2513 | yield* log("accepted", { |
| 2514 | signatureVerified: signature !== null, |
| 2515 | missing: rendered.missing, |
| 2516 | renderedPrompt: rendered.prompt, |
| 2517 | }); |
| 2518 | return outcome; |
| 2519 | }), |
| 2520 | ).pipe(Effect.onError(() => release)); |
| 2521 | if (typeof admitted !== "string") { |
| 2522 | yield* release; |
| 2523 | if (admitted.log) yield* log("rate_limited"); |
| 2524 | yield* observe(admitted.outcome); |
| 2525 | return yield* releaseClaim({ |
| 2526 | _tag: "rate_limited", |
| 2527 | outcome: admitted.outcome, |
| 2528 | } as const); |
| 2529 | } |
| 2530 | if (admitted === "duplicate") { |
| 2531 | yield* release; |
| 2532 | yield* observe("duplicate"); |
| 2533 | return yield* releaseClaim({ |
| 2534 | _tag: "accepted", |
| 2535 | deliveryId, |
| 2536 | outcome: "duplicate", |
| 2537 | } as const); |
| 2538 | } |
| 2539 | yield* observe("accepted"); |
| 2540 | const permit = yield* webhookPermit(task.id); |
| 2541 | yield* runWebhookDelivery(task, deliveryId, rendered.prompt).pipe( |
| 2542 | permit.withPermits(1), |
| 2543 | Effect.ensuring(release), |
| 2544 | Effect.forkIn(serviceScope), |
| 2545 | ); |
| 2546 | // Dispatch owns the accepted delivery now; retries must keep its claim. |
| 2547 | claimAcquired = false; |
| 2548 | return { _tag: "accepted" as const, deliveryId, outcome: "accepted" as const }; |
| 2549 | }).pipe( |
| 2550 | Effect.onError(() => |
| 2551 | Effect.all([ |
| 2552 | releaseClaim(undefined), |
| 2553 | (request.relayDeliveryId === undefined || claimAcquired |
| 2554 | ? releaseReplay(deliveryId) |
| 2555 | : Effect.void |
| 2556 | ).pipe(Effect.ignore), |
| 2557 | ]), |
| 2558 | ), |
| 2559 | ), |
| 2560 | ); |
| 2561 | }); |
| 2562 | |
| 2563 | const dispatchPending = Effect.fn("ScheduledTaskService.dispatchPendingWebhooks")(function* () { |
| 2564 | const now = DateTime.toEpochMillis(yield* localNow); |
| 2565 | if (yield* pendingWebhooks.isIdle(now)) return; |
| 2566 | const readVersion = yield* pendingWebhooks.version; |
| 2567 | yield* sql`UPDATE scheduled_task_webhook_pending SET state = 'dispatching' WHERE state = 'pending' AND due_at <= ${now}`; |
| 2568 | const tasks = yield* sql<{ task_id: ScheduledTaskId }>` |
| 2569 | SELECT DISTINCT task_id FROM scheduled_task_webhook_pending WHERE state IN ('dispatching', 'running', 'completed', 'failed') |
| 2570 | `.pipe(NodeSqliteClient.readOnly); |
| 2571 | // Workers can leave rows behind to retry, so keep polling while any are claimed. |
| 2572 | if (tasks.length === 0) { |
| 2573 | const [next] = yield* sql<{ due_at: number | null }>` |
| 2574 | SELECT MIN(due_at) AS due_at FROM scheduled_task_webhook_pending WHERE state = 'pending' |
| 2575 | `.pipe(NodeSqliteClient.readOnly); |
| 2576 | yield* pendingWebhooks.record(readVersion, next?.due_at ?? Infinity, now); |
| 2577 | } |
| 2578 | yield* Effect.forEach( |
| 2579 | tasks, |
| 2580 | ({ task_id: taskId }) => { |
| 2581 | // One worker per task waits on its permit, then drains persisted rows in order. |
| 2582 | // Other tasks remain independent and ticks never reload an active task's payloads. |
| 2583 | const dispatch = Effect.gen(function* () { |
| 2584 | while (true) { |
| 2585 | const [entry] = yield* sql<{ |
| 2586 | delivery_id: string; |
| 2587 | task_json: string; |
| 2588 | request_json: string; |
| 2589 | state: string; |
| 2590 | }>` |
| 2591 | SELECT delivery_id, task_json, request_json, state FROM scheduled_task_webhook_pending |
| 2592 | WHERE task_id = ${taskId} AND state IN ('dispatching', 'running', 'completed', 'failed') ORDER BY due_at, rowid LIMIT 1 |
| 2593 | `.pipe(NodeSqliteClient.readOnly); |
| 2594 | if (entry === undefined) return; |
| 2595 | const remove = sql`DELETE FROM scheduled_task_webhook_pending WHERE delivery_id = ${entry.delivery_id}`; |
| 2596 | if (entry.state === "completed" || entry.state === "failed") { |
| 2597 | if (entry.state === "failed") { |
| 2598 | yield* markDeliveryFailed(entry.delivery_id, "The run failed to start."); |
| 2599 | yield* releaseFailedDelivery(entry.delivery_id); |
| 2600 | } |
| 2601 | yield* remove; |
| 2602 | continue; |
| 2603 | } |
| 2604 | const prepared = yield* Effect.gen(function* () { |
| 2605 | // A running row was already sent, so recovery keeps the prompt it was sent with. |
| 2606 | const current = entry.state === "dispatching" ? yield* findTask(taskId) : null; |
| 2607 | const snapshot = yield* decodePendingTask(entry.task_json); |
| 2608 | // The snapshot keeps the identity runTask checks; the prompt is the one saved now. |
| 2609 | const task = |
| 2610 | current?.createdAt === snapshot.createdAt && current.prompt !== snapshot.prompt |
| 2611 | ? { ...snapshot, prompt: current.prompt } |
| 2612 | : snapshot; |
| 2613 | const request = yield* decodePendingRequest(entry.request_json); |
| 2614 | const frameId = Buffer.from(yield* random.randomBytes(6)).toString("hex"); |
| 2615 | return { |
| 2616 | task, |
| 2617 | taskJson: task === snapshot ? null : yield* encodePendingTask(task), |
| 2618 | rendered: renderWebhookPrompt(task.prompt, request, frameId), |
| 2619 | }; |
| 2620 | }).pipe(Effect.exit); |
| 2621 | if (Exit.isFailure(prepared)) { |
| 2622 | if (Cause.hasInterruptsOnly(prepared.cause)) |
| 2623 | return yield* Effect.failCause(prepared.cause); |
| 2624 | yield* markDeliveryFailed( |
| 2625 | entry.delivery_id, |
| 2626 | "The pending webhook payload could not be prepared.", |
| 2627 | ); |
| 2628 | yield* releaseFailedDelivery(entry.delivery_id); |
| 2629 | yield* remove; |
| 2630 | continue; |
| 2631 | } |
| 2632 | const { task, taskJson, rendered } = prepared.value; |
| 2633 | if (rendered.prompt.trim().length > PROVIDER_SEND_TURN_MAX_INPUT_CHARS) { |
| 2634 | yield* markDeliveryFailed(entry.delivery_id, PROMPT_TOO_LONG); |
| 2635 | yield* releaseFailedDelivery(entry.delivery_id); |
| 2636 | } else { |
| 2637 | if (taskJson !== null) |
| 2638 | yield* sql`UPDATE scheduled_task_webhook_pending SET task_json = ${taskJson} WHERE delivery_id = ${entry.delivery_id}`; |
| 2639 | yield* sql`UPDATE scheduled_task_webhook_deliveries SET outcome = 'accepted', error = NULL, rendered_prompt = ${rendered.prompt.slice(0, WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT)}, missing_fields_json = ${encodeMissingFieldsJson(rendered.missing)} WHERE delivery_id = ${entry.delivery_id}`; |
| 2640 | if (!(yield* runWebhookDelivery(task, entry.delivery_id, rendered.prompt))) return; |
| 2641 | } |
| 2642 | yield* remove; |
| 2643 | } |
| 2644 | }); |
| 2645 | // The worker owns its reservation, so an interrupt anywhere still releases it. |
| 2646 | return Effect.acquireUseRelease( |
| 2647 | Ref.modify(pendingInFlight, (active) => |
| 2648 | active.has(taskId) |
| 2649 | ? ([false, active] as const) |
| 2650 | : ([true, new Set(active).add(taskId)] as const), |
| 2651 | ), |
| 2652 | (reserved) => |
| 2653 | reserved |
| 2654 | ? webhookPermit(taskId).pipe( |
| 2655 | Effect.flatMap((permit) => dispatch.pipe(permit.withPermits(1))), |
| 2656 | Effect.catch((cause) => |
| 2657 | Effect.logWarning("Pending webhook dispatch failed", { cause }), |
| 2658 | ), |
| 2659 | Effect.withSpan("ScheduledTaskService.dispatchPendingWebhook", { |
| 2660 | root: true, |
| 2661 | }), |
| 2662 | ) |
| 2663 | : Effect.void, |
| 2664 | (reserved) => |
| 2665 | reserved |
| 2666 | ? Ref.update(pendingInFlight, (active) => { |
| 2667 | const next = new Set(active); |
| 2668 | next.delete(taskId); |
| 2669 | return next; |
| 2670 | }) |
| 2671 | : Effect.void, |
| 2672 | ).pipe(Effect.forkIn(serviceScope)); |
| 2673 | }, |
| 2674 | { discard: true }, |
| 2675 | ); |
| 2676 | }); |
| 2677 | yield* scheduler.register("webhook-coalescing", dispatchPending()); |
| 2678 | |
| 2679 | return ScheduledTaskService.of({ |
| 2680 | list, |
| 2681 | subscribeList, |
| 2682 | upsert, |
| 2683 | setEnabled, |
| 2684 | delete: deleteTask, |
| 2685 | deleteForThread, |
| 2686 | moveThreadTasks, |
| 2687 | runNow, |
| 2688 | rotateWebhookToken, |
| 2689 | listWebhookDeliveries, |
| 2690 | getWebhookDelivery, |
| 2691 | triggerWebhook, |
| 2692 | }); |
| 2693 | }), |
| 2694 | ); |