@devries-cameron20 / t3code Mirror · read-only / commits / 2ebe280e56
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 20:45 UTC

feat(scheduled-tasks): replay dedupe and coalescing for webhook deliveries (#22234)

Julius Marminge committed Oct 20, 2026 at 13:49 UTC 2ebe280e5602f23fc2742297da3d3b6f2b1b8fca
20 files changed +3123 -326
apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx
+42
@@ -716,6 +716,13 @@ function TaskForm({
716 Dialog.alert("Invalid filter", parsedFilters.error);
717 return;
718 }
719 + if (draft.schedule.mode === "webhook" && schedule === null) {
720 + Dialog.alert(
721 + "Invalid webhook settings",
722 + "Each dedupe key must contain a valid request placeholder. The coalesce window must be whole seconds from 10 to 3600, or blank.",
723 + );
724 + return;
725 + }
726 if (
727 !draft.title.trim() ||
728 !draft.prompt.trim() ||
@@ -1060,6 +1067,41 @@ function TaskForm({
1067 setDraft({ ...draft, schedule: { ...draft.schedule, filters } })
1068 }
1069 />
1070 + <FormField
1071 + label="Replay key template"
1072 + value={draft.schedule.replayKey}
1073 + placeholder="{{headers.delivery-id}}"
1074 + disabled={saving}
1075 + borderTop
1076 + onChange={(replayKey) =>
1077 + setDraft({ ...draft, schedule: { ...draft.schedule, replayKey } })
1078 + }
1079 + />
1080 + <FormField
1081 + label="Coalesce key template"
1082 + value={draft.schedule.coalesceKey}
1083 + placeholder="{{body.id}}"
1084 + disabled={saving}
1085 + borderTop
1086 + onChange={(coalesceKey) =>
1087 + setDraft({ ...draft, schedule: { ...draft.schedule, coalesceKey } })
1088 + }
1089 + />
1090 + <FormField
1091 + label="Coalesce window (seconds, 10–3600)"
1092 + value={draft.schedule.coalesceWindowSeconds}
1093 + placeholder="120"
1094 + keyboardType="number-pad"
1095 + disabled={saving}
1096 + borderTop
1097 + onChange={(coalesceWindowSeconds) =>
1098 + setDraft({ ...draft, schedule: { ...draft.schedule, coalesceWindowSeconds } })
1099 + }
1100 + />
1101 + <Text className="px-4 pb-3 text-sm text-foreground-muted">
1102 + Replay keys suppress repeats for 7 days. Coalescing runs the latest payload when the
1103 + window from the first delivery closes. Empty keys run normally.
1104 + </Text>
1105 </>
1106 ) : (
1107 <>
apps/mobile/src/features/settings/scheduledTaskDraft.test.ts
+43 -1
@@ -30,6 +30,37 @@ describe("scheduleDraftForTask", () => {
30 expect(scheduleFromDraft(scheduleDraftForTask({ schedule }))).toEqual(schedule);
31 });
32
33 + it("round-trips dedupe settings and rejects invalid windows", () => {
34 + const dedupe = {
35 + replayKey: "{{headers.delivery-id}}",
36 + coalesceKey: "{{body.id}}",
37 + coalesceWindowSeconds: 3600,
38 + };
39 + const draft = scheduleDraftForTask({ schedule: { type: "webhook", signature: null, dedupe } });
40 + expect(scheduleFromDraft(draft)).toMatchObject({ dedupe });
41 + expect(
42 + scheduleFromDraft({
43 + ...draft,
44 + replayKey: " \n ",
45 + coalesceKey: " \t ",
46 + coalesceWindowSeconds: "",
47 + }),
48 + ).toMatchObject({ dedupe: {} });
49 + expect(
50 + scheduleFromDraft({
51 + ...draft,
52 + replayKey: ` ${dedupe.replayKey} `,
53 + coalesceKey: `\t${dedupe.coalesceKey}\n`,
54 + }),
55 + ).toMatchObject({ dedupe });
56 + expect(
57 + scheduleFromDraft({ ...draft, replayKey: "", coalesceKey: "", coalesceWindowSeconds: "" }),
58 + ).toMatchObject({ dedupe: {} });
59 + for (const coalesceWindowSeconds of ["9", "3601", "1.5", "abc"]) {
60 + expect(scheduleFromDraft({ ...draft, coalesceWindowSeconds })).toBeNull();
61 + }
62 + });
63 +
64 it("round-trips a webhook schedule without a signature", () => {
65 const draft = scheduleDraftForTask({ schedule: { type: "webhook", signature: null } });
66 expect(draft.mode).toBe("webhook");
@@ -37,6 +68,7 @@ describe("scheduleDraftForTask", () => {
68 type: "webhook",
69 signature: null,
70 maxDeliveryAgeMinutes: null,
71 + dedupe: {},
72 });
73 });
74
@@ -50,6 +82,7 @@ describe("scheduleDraftForTask", () => {
82 expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: "0" })).toBeNull();
83 expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: "" })).toMatchObject({
84 maxDeliveryAgeMinutes: null,
85 + dedupe: {},
86 });
87 });
88
@@ -62,7 +95,7 @@ describe("scheduleDraftForTask", () => {
95 const saved = scheduleFromDraft(
96 scheduleDraftForTask({ schedule: { type: "webhook", signature } }),
97 );
65 - expect(saved).toEqual({ type: "webhook", signature, maxDeliveryAgeMinutes: null });
98 + expect(saved).toEqual({ type: "webhook", signature, maxDeliveryAgeMinutes: null, dedupe: {} });
99 expect(saved?.type === "webhook" && saved.signature && "secret" in saved.signature).toBe(false);
100 });
101
@@ -369,3 +402,12 @@ describe("scheduled task model defaults", () => {
402 ).toBeNull();
403 });
404 });
405 +
406 +it("rejects dedupe keys without valid request placeholders", () => {
407 + expect(
408 + scheduleFromDraft({ ...DEFAULT_SCHEDULE, mode: "webhook", replayKey: "literal" }),
409 + ).toBeNull();
410 + expect(
411 + scheduleFromDraft({ ...DEFAULT_SCHEDULE, mode: "webhook", coalesceKey: "{{body.bad" }),
412 + ).toBeNull();
413 +});
apps/mobile/src/features/settings/scheduledTaskDraft.ts
+38
@@ -1,3 +1,4 @@
1 +import { isWebhookDedupeTemplate } from "@t3tools/contracts";
2 import type {
3 ModelSelection,
4 ServerConfig,
@@ -56,6 +57,9 @@ export type ScheduleDraft = {
57 readonly intervalMinutes: string;
58 readonly signature: ScheduledTaskWebhookSignature | null;
59 readonly maxDeliveryAgeMinutes: string;
60 + readonly replayKey: string;
61 + readonly coalesceKey: string;
62 + readonly coalesceWindowSeconds: string;
63 readonly filters: ReadonlyArray<WebhookFilterDraft>;
64 /** The filter rows the form opened with. */
65 readonly savedFilters: ReadonlyArray<WebhookFilterDraft>;
@@ -68,6 +72,9 @@ export const DEFAULT_SCHEDULE: ScheduleDraft = {
72 intervalMinutes: "15",
73 signature: null,
74 maxDeliveryAgeMinutes: "",
75 + replayKey: "",
76 + coalesceKey: "",
77 + coalesceWindowSeconds: "",
78 filters: [],
79 savedFilters: [],
80 };
@@ -93,6 +100,12 @@ export function scheduleDraftForTask(task: Pick<ScheduledTask, "schedule">): Sch
100 ...DEFAULT_SCHEDULE,
101 mode: "webhook",
102 signature: task.schedule.signature,
103 + replayKey: task.schedule.dedupe?.replayKey ?? "",
104 + coalesceKey: task.schedule.dedupe?.coalesceKey ?? "",
105 + coalesceWindowSeconds:
106 + task.schedule.dedupe?.coalesceWindowSeconds === undefined
107 + ? ""
108 + : String(task.schedule.dedupe.coalesceWindowSeconds),
109 maxDeliveryAgeMinutes:
110 task.schedule.maxDeliveryAgeMinutes == null
111 ? ""
@@ -107,6 +120,19 @@ export function scheduleFromDraft(draft: ScheduleDraft): ScheduledTaskUpsertSche
120 if (draft.mode === "webhook") {
121 const maxDeliveryAgeMinutes = parseMaxDeliveryAge(draft.maxDeliveryAgeMinutes);
122 if (maxDeliveryAgeMinutes === undefined) return null;
123 + const replayKey = draft.replayKey.trim();
124 + const coalesceKey = draft.coalesceKey.trim();
125 + if ([replayKey, coalesceKey].some((key) => key && !isWebhookDedupeTemplate(key))) return null;
126 + const coalesceWindowSeconds = draft.coalesceWindowSeconds?.trim()
127 + ? Number(draft.coalesceWindowSeconds)
128 + : undefined;
129 + if (
130 + coalesceWindowSeconds !== undefined &&
131 + (!Number.isInteger(coalesceWindowSeconds) ||
132 + coalesceWindowSeconds < 10 ||
133 + coalesceWindowSeconds > 3600)
134 + )
135 + return null;
136 const parsedFilters = parseWebhookFilterDrafts(draft.filters);
137 if (!("filters" in parsedFilters)) return null;
138 // No secret is sent, so the server keeps the stored one.
@@ -121,6 +147,15 @@ export function scheduleFromDraft(draft: ScheduleDraft): ScheduledTaskUpsertSche
147 prefix: draft.signature.prefix,
148 },
149 maxDeliveryAgeMinutes,
150 + ...(replayKey || coalesceKey || coalesceWindowSeconds !== undefined
151 + ? {
152 + dedupe: {
153 + ...(replayKey ? { replayKey } : {}),
154 + ...(coalesceKey ? { coalesceKey } : {}),
155 + ...(coalesceWindowSeconds === undefined ? {} : { coalesceWindowSeconds }),
156 + },
157 + }
158 + : { dedupe: {} }),
159 // Only edited filters are sent; the server keeps the stored ones otherwise.
160 ...(webhookFilterDraftsChanged(draft.savedFilters, draft.filters)
161 ? { filters: parsedFilters.filters }
@@ -179,6 +214,9 @@ function draftSignature(draft: ScheduledTaskDraft): string {
214 [...draft.schedule.weekdays].sort((a, b) => a - b),
215 draft.schedule.intervalMinutes,
216 draft.schedule.maxDeliveryAgeMinutes,
217 + draft.schedule.replayKey,
218 + draft.schedule.coalesceKey,
219 + draft.schedule.coalesceWindowSeconds,
220 draft.schedule.filters.map(({ path, op, value }) => [path, op, value]),
221 draft.workspace,
222 draft.baseRef,
apps/server/src/mcp/toolkits/orchestrator/tools.ts
+2 -2
@@ -116,7 +116,7 @@ const WorkflowStopTool = Tool.make("t3_workflow_stop", {
116
117 export const ScheduleTaskTool = Tool.make("schedule_task", {
118 description:
119 - "Create persistent work in the app scheduler that runs even when no turn is active. Pass schedule as a STRUCTURED OBJECT, never JSON text. Timers: {type:'interval', everyMs:3600000} is hourly; {type:'fixed_time', timeOfDay:'09:00', weekdays:[1,2,3,4,5]} is weekday mornings; report the returned nextRunAt. Webhooks: {type:'webhook'} runs once per request to a generated URL. The run sees the request ONLY through prompt placeholders: {{body.path}} (e.g. {{body.action}}, {{body.release.tag_name}}), {{headers.name}}, {{query.name}}, {{body}}, or {{request}} (method, headers with credentials redacted, and body). Each value arrives framed as untrusted request data, and $skill, @file or /command mentions inside it do not act; without a signature, anyone with the URL can trigger a run. For a sender that signs requests, first call request_secret so the user enters the secret privately (never ask for it in chat or invent one), then set signature with the returned secretRef, e.g. GitHub: {type:'webhook', signature:{header:'x-hub-signature-256', encoding:'hex', prefix:'sha256=', secretRef}}. To run only for some requests, add filters, all of which must match, e.g. filters:[{path:'headers.x-github-event', op:'equals', value:'pull_request'}, {path:'body.action', op:'in', value:['opened','reopened']}]; ops are equals, notEquals, in, startsWith and exists. The result's webhookUrl is the public URL to give the user; if it is absent, this environment has no T3 Connect managed tunnel, so tell the user to enable T3 Connect remote access rather than sharing a path. Omit projectId for this thread's project. In this thread's project, runs post into THIS thread by default (bindToCurrentThread=true), which suits an orchestrator that sees every trigger, delegates work, and can dedupe against what is in flight; use false only when the user wants a fresh top-level thread per run. Elsewhere each run launches a fresh thread. Provider, model, and runtime settings inherit from this thread, or from the project default when there is no calling thread.",
119 + "Create persistent work in the app scheduler that runs even when no turn is active. Pass schedule as a STRUCTURED OBJECT, never JSON text. Timers: {type:'interval', everyMs:3600000} is hourly; {type:'fixed_time', timeOfDay:'09:00', weekdays:[1,2,3,4,5]} is weekday mornings; report the returned nextRunAt. Webhook dedupe accepts replayKey and coalesceKey templates with coalesceWindowSeconds (10–3600, default 120); replay keys expire after 7 days and coalescing keeps the latest payload in a window measured from its first delivery. Webhooks: {type:'webhook'} runs once per request to a generated URL. The run sees the request ONLY through prompt placeholders: {{body.path}} (e.g. {{body.action}}, {{body.release.tag_name}}), {{headers.name}}, {{query.name}}, {{body}}, or {{request}} (method, headers with credentials redacted, and body). Each value arrives framed as untrusted request data, and $skill, @file or /command mentions inside it do not act; without a signature, anyone with the URL can trigger a run. For a sender that signs requests, first call request_secret so the user enters the secret privately (never ask for it in chat or invent one), then set signature with the returned secretRef, e.g. GitHub: {type:'webhook', signature:{header:'x-hub-signature-256', encoding:'hex', prefix:'sha256=', secretRef}}. To run only for some requests, add filters, all of which must match, e.g. filters:[{path:'headers.x-github-event', op:'equals', value:'pull_request'}, {path:'body.action', op:'in', value:['opened','reopened']}]; ops are equals, notEquals, in, startsWith and exists. The result's webhookUrl is the public URL to give the user; if it is absent, this environment has no T3 Connect managed tunnel, so tell the user to enable T3 Connect remote access rather than sharing a path. Omit projectId for this thread's project. In this thread's project, runs post into THIS thread by default (bindToCurrentThread=true), which suits an orchestrator that sees every trigger, delegates work, and can dedupe against what is in flight; use false only when the user wants a fresh top-level thread per run. Elsewhere each run launches a fresh thread. Provider, model, and runtime settings inherit from this thread, or from the project default when there is no calling thread.",
120 parameters: OrchestratorMcpScheduleTaskInput,
121 success: OrchestratorMcpScheduleTaskResult,
122 failure: OrchestratorMcpFailure,
@@ -143,7 +143,7 @@ const ListScheduledTasksTool = Tool.make("list_scheduled_tasks", {
143
144 const UpdateScheduledTaskTool = Tool.make("update_scheduled_task", {
145 description:
146 - "Update an existing scheduled task by scheduledTaskId (from list_scheduled_tasks). Only the provided fields change; omit a field to leave it as-is. Use enabled=false to pause a task without deleting it. Set bindToCurrentThread to move the task between posting into this thread and launching a fresh thread per run. A webhook schedule without filters keeps the stored filters; pass filters to replace them, or filters:[] to clear them.",
146 + "Update an existing scheduled task by scheduledTaskId (from list_scheduled_tasks). Webhook schedule dedupe accepts replayKey, coalesceKey, and coalesceWindowSeconds (10–3600, default 120) to suppress replays for 7 days and retain the latest payload in each fixed window. Only the provided fields change; omit a field to leave it as-is. Use enabled=false to pause a task without deleting it. Set bindToCurrentThread to move the task between posting into this thread and launching a fresh thread per run. A webhook schedule without filters keeps the stored filters; pass filters to replace them, or filters:[] to clear them.",
147 parameters: OrchestratorMcpUpdateScheduledTaskInput,
148 success: OrchestratorMcpScheduleTaskResult,
149 failure: OrchestratorMcpFailure,
apps/server/src/persistence/Migrations.ts
+2
@@ -69,6 +69,7 @@ import Migration0063 from "./Migrations/063_DropUnusedV1Projections.ts";
69 import Migration0064 from "./Migrations/064_CanonicalizeThreadPullRequestLinks.ts";
70 import Migration0065 from "./Migrations/065_CodexUsageNativeTurnIndex.ts";
71 import Migration0066 from "./Migrations/066_RepairClientSettlementTimestamps.ts";
72 +import Migration0067 from "./Migrations/067_WebhookDedupe.ts";
73
74 export const migrationEntries = [
75 [1, "OrchestrationEvents", Migration0001],
@@ -138,6 +139,7 @@ export const migrationEntries = [
139 [64, "CanonicalizeThreadPullRequestLinks", Migration0064],
140 [65, "CodexUsageNativeTurnIndex", Migration0065],
141 [66, "RepairClientSettlementTimestamps", Migration0066],
142 + [67, "WebhookDedupe", Migration0067],
143 ] as const;
144
145 export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const);
apps/server/src/persistence/Migrations/055_OrchestrationV2.test.ts
+3 -1
@@ -15,7 +15,7 @@ layer("055_OrchestrationV2", (it) => {
15 Effect.sync(() => {
16 assert.deepStrictEqual(
17 migrationEntries.map(([id]) => id),
18 - Array.from({ length: 66 }, (_, index) => index + 1),
18 + Array.from({ length: 67 }, (_, index) => index + 1),
19 );
20 }),
21 );
@@ -40,6 +40,7 @@ layer("055_OrchestrationV2", (it) => {
40 [64, "CanonicalizeThreadPullRequestLinks"],
41 [65, "CodexUsageNativeTurnIndex"],
42 [66, "RepairClientSettlementTimestamps"],
43 + [67, "WebhookDedupe"],
44 ]);
45 assert.deepStrictEqual(yield* runMigrations(), []);
46
@@ -72,6 +73,7 @@ layer("055_OrchestrationV2", (it) => {
73 { migration_id: 64, name: "CanonicalizeThreadPullRequestLinks" },
74 { migration_id: 65, name: "CodexUsageNativeTurnIndex" },
75 { migration_id: 66, name: "RepairClientSettlementTimestamps" },
76 + { migration_id: 67, name: "WebhookDedupe" },
77 ]);
78
79 const tables = yield* sql<{ readonly name: string }>`
apps/server/src/persistence/Migrations/067_WebhookDedupe.ts new
+29
@@ -0,0 +1,29 @@
1 +import * as Effect from "effect/Effect";
2 +import * as SqlClient from "effect/sql/SqlClient";
3 +
4 +export default Effect.gen(function* () {
5 + const sql = yield* SqlClient.SqlClient;
6 + yield* sql`ALTER TABLE scheduled_tasks ADD COLUMN last_run_webhook_delivery_id TEXT`;
7 + yield* sql`CREATE TABLE scheduled_task_webhook_replay_keys (
8 + task_id TEXT NOT NULL REFERENCES scheduled_tasks(task_id) ON DELETE CASCADE,
9 + replay_key TEXT NOT NULL,
10 + delivery_id TEXT NOT NULL,
11 + expires_at INTEGER NOT NULL,
12 + PRIMARY KEY (task_id, replay_key)
13 + )`;
14 + yield* sql`CREATE INDEX idx_webhook_replay_expiry ON scheduled_task_webhook_replay_keys(expires_at)`;
15 + yield* sql`CREATE INDEX idx_webhook_replay_delivery ON scheduled_task_webhook_replay_keys(delivery_id)`;
16 + yield* sql`CREATE TABLE scheduled_task_webhook_pending (
17 + delivery_id TEXT PRIMARY KEY,
18 + task_id TEXT NOT NULL REFERENCES scheduled_tasks(task_id) ON DELETE CASCADE,
19 + coalesce_key TEXT NOT NULL,
20 + due_at INTEGER NOT NULL,
21 + state TEXT NOT NULL DEFAULT 'pending',
22 + task_json TEXT NOT NULL,
23 + request_json TEXT NOT NULL,
24 + delivery_ids_json TEXT NOT NULL
25 + )`;
26 + yield* sql`CREATE INDEX idx_webhook_pending_due ON scheduled_task_webhook_pending(state, due_at)`;
27 + yield* sql`CREATE UNIQUE INDEX idx_webhook_pending_key
28 + ON scheduled_task_webhook_pending(task_id, coalesce_key) WHERE state = 'pending'`;
29 +});
apps/server/src/persistence/reconcileV2PreviewMigration.test.ts
+2
@@ -46,6 +46,7 @@ describe("V2 preview upgrade", () => {
46 [64, "CanonicalizeThreadPullRequestLinks"],
47 [65, "CodexUsageNativeTurnIndex"],
48 [66, "RepairClientSettlementTimestamps"],
49 + [67, "WebhookDedupe"],
50 ]);
51 assert.deepStrictEqual(yield* runMigrations(), []);
52 assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports);
@@ -135,6 +136,7 @@ describe("V2 preview upgrade", () => {
136 [64, "CanonicalizeThreadPullRequestLinks"],
137 [65, "CodexUsageNativeTurnIndex"],
138 [66, "RepairClientSettlementTimestamps"],
139 + [67, "WebhookDedupe"],
140 ]);
141 }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))),
142 );
apps/server/src/scheduledTasks/ScheduledTaskService.ts
+581 -161
@@ -29,6 +29,7 @@ import * as Crypto from "effect/Crypto";
29 import * as Data from "effect/Data";
30 import * as DateTime from "effect/DateTime";
31 import * as Effect from "effect/Effect";
32 +import * as Exit from "effect/Exit";
33 import * as Duration from "effect/Duration";
34 import * as Metric from "effect/Metric";
35 import * as Layer from "effect/Layer";
@@ -54,14 +55,30 @@ import {
55 renderWebhookPrompt,
56 type WebhookRequest,
57 } from "./webhookTemplate.ts";
58 +import {
59 + acceptWebhookDedupe,
60 + acceptImmediateWebhookDedupe,
61 + releaseWebhookReplay,
62 + decodePendingTask,
63 + decodePendingRequest,
64 + dedupeError,
65 +} from "./webhookDedupe.ts";
66 import { describeFailedWebhookFilter, findFailedWebhookFilter } from "./webhookFilter.ts";
67 import { constantTimeEquals, verifyWebhookSignature } from "./webhookVerification.ts";
68
69 export const WEBHOOK_ROUTE_PREFIX = "/api/hooks";
70 +const isScheduledTaskError = Schema.is(ScheduledTaskError);
71 +const PROMPT_TOO_LONG = "The filled-in prompt is too long.";
72 const WEBHOOK_DELIVERY_RETENTION = 50;
62 -/** Kept apart from the rest, so a stream of filtered requests can't evict real deliveries. */
73 +const WEBHOOK_DEDUPE_DELIVERY_RETENTION = 20;
74 const WEBHOOK_FILTERED_DELIVERY_RETENTION = 20;
64 -/** The outcomes clients decoded before outcomes became open-ended. */
75 +const WEBHOOK_DEDUPE_LOG_PER_MINUTE = 10;
76 +const WEBHOOK_FILTERED_LOG_PER_MINUTE = 10;
77 +const WEBHOOK_DELIVERY_LOG_BODY_LIMIT = 64 * 1024;
78 +const WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT = 64 * 1024;
79 +const WEBHOOK_MAX_QUEUED_PER_TASK = 20;
80 +/** Accepted deliveries per task per minute, enforced here as well as on the relay because the tunnel hostname is public too. */
81 +const WEBHOOK_RATE_LIMIT_PER_MINUTE = 60;
82 const LEGACY_DELIVERY_OUTCOMES: ReadonlySet<string> = new Set([
83 "accepted",
84 "dispatch_failed",
@@ -70,16 +87,6 @@ const LEGACY_DELIVERY_OUTCOMES: ReadonlySet<string> = new Set([
87 "rate_limited",
88 "expired",
89 ]);
73 -/**
74 - * Filtered requests take no run slot, so this bounds the log writes they
75 - * cause instead; past it a filtered request is only counted in metrics.
76 - */
77 -const WEBHOOK_FILTERED_LOG_PER_MINUTE = 10;
78 -const WEBHOOK_DELIVERY_LOG_BODY_LIMIT = 64 * 1024;
79 -const WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT = 64 * 1024;
80 -const WEBHOOK_MAX_QUEUED_PER_TASK = 20;
81 -/** Accepted deliveries per task per minute, enforced here as well as on the relay because the tunnel hostname is public too. */
82 -const WEBHOOK_RATE_LIMIT_PER_MINUTE = 60;
90
91 interface WebhookOrigin {
92 readonly relayHookBaseUrl: string | null;
@@ -111,7 +118,7 @@ class WebhookDeliverySkipped extends Data.TaggedError("WebhookDeliverySkipped")<
118 }> {}
119
120 interface RateWindow {
114 - readonly accepted: ReadonlyArray<number>;
121 + readonly accepted: ReadonlyArray<{ readonly at: number }>;
122 readonly rejectedLogged: boolean;
123 }
124
@@ -126,6 +133,7 @@ export interface WebhookTriggerRequest extends WebhookRequest {
133 export type WebhookDeliveryOutcome =
134 | "accepted"
135 | "duplicate"
136 + | "coalesced"
137 | "prompt_too_long"
138 | "queue_full"
139 | "rate_limited"
@@ -140,7 +148,7 @@ export type WebhookTriggerResult =
148 | {
149 readonly _tag: "accepted";
150 readonly deliveryId: ScheduledTaskWebhookDeliveryId;
143 - readonly outcome: "accepted" | "duplicate" | "prompt_too_long" | "filtered";
151 + readonly outcome: "accepted" | "duplicate" | "coalesced" | "prompt_too_long" | "filtered";
152 }
153 | { readonly _tag: "not_found" }
154 | { readonly _tag: "rejected_signature" }
@@ -211,6 +219,7 @@ interface WebhookDeliveryRow {
219 readonly signature_verified: number;
220 readonly missing_fields_json: string;
221 readonly rendered_prompt: string | null;
222 + readonly pending_window: number;
223 readonly error: string | null;
224 }
225
@@ -405,6 +414,13 @@ export const layer = Layer.effect(
414 // finish late release their own count, never a recreated task's.
415 const webhookQueued = yield* Ref.make<ReadonlyMap<string, number>>(new Map());
416 const webhookRateWindows = yield* Ref.make<ReadonlyMap<ScheduledTaskId, RateWindow>>(new Map());
417 + const webhookRejectedRateWindows = yield* Ref.make<ReadonlyMap<ScheduledTaskId, RateWindow>>(
418 + new Map(),
419 + );
420 + const webhookDedupeLogWindows = yield* Ref.make<
421 + ReadonlyMap<ScheduledTaskId, ReadonlyArray<number>>
422 + >(new Map());
423 + const pendingInFlight = yield* Ref.make<ReadonlySet<string>>(new Set());
424 const webhookFilteredLogWindows = yield* Ref.make<
425 ReadonlyMap<ScheduledTaskId, ReadonlyArray<number>>
426 >(new Map());
@@ -627,11 +643,12 @@ export const layer = Layer.effect(
643 ),
644 );
645
630 - const markRunning = (id: ScheduledTaskId, startedAtIso: string) =>
646 + const markRunning = (id: ScheduledTaskId, startedAtIso: string, webhookDeliveryId?: string) =>
647 sql<{ task_id: string }>`
648 UPDATE scheduled_tasks
649 SET updated_at = ${startedAtIso},
650 last_run_at = ${startedAtIso},
651 + last_run_webhook_delivery_id = ${webhookDeliveryId ?? null},
652 last_run_status = 'running',
653 last_run_error = NULL
654 WHERE task_id = ${id}
@@ -673,9 +690,10 @@ export const layer = Layer.effect(
690 // markCompleted, write a full terminal record so runDueTasks neither skips
691 // the task forever (it filters out 'running' rows) nor re-fires it
692 // immediately: the dispatch may already have gone out, so next_run_at must
676 - // advance and run_count must count the attempt.
693 + // advance. Pending webhook attempts count only with their completion state;
694 + // other attempts count here.
695 const releaseStuckRun = Effect.fn("ScheduledTaskService.releaseStuckRun")(
678 - function* (task: ScheduledTask, message: string) {
696 + function* (task: ScheduledTask, message: string, pendingDeliveryId?: string) {
697 const now = yield* localNow;
698 const reread = yield* Effect.result(findTask(task.id));
699 if (Result.isSuccess(reread) && reread.success === null) return;
@@ -686,12 +704,15 @@ export const layer = Layer.effect(
704 last_run_error = ${message},
705 next_run_at = ${nextRunAt(source, now)},
706 updated_at = ${iso(now)},
689 - run_count = run_count + 1
707 + run_count = run_count + CASE WHEN EXISTS (
708 + SELECT 1 FROM scheduled_task_webhook_pending
709 + WHERE delivery_id = ${pendingDeliveryId ?? null} AND state = 'running'
710 + ) THEN 0 ELSE 1 END
711 WHERE task_id = ${task.id} AND last_run_status = 'running'
712 `;
713 yield* notifyChanged;
714 },
694 - (effect, task, _message) =>
715 + (effect, task, _message, _pendingDeliveryId) =>
716 effect.pipe(
717 Effect.catch((cause) =>
718 Effect.logWarning("Could not release stuck schedule task run", {
@@ -705,7 +726,11 @@ export const layer = Layer.effect(
726 const runTask = Effect.fn("ScheduledTaskService.runTask")(function* (
727 task: ScheduledTask,
728 trigger: "scheduled" | "manual" | "webhook",
708 - webhook?: { readonly deliveryId: string; readonly prompt: string },
729 + webhook?: {
730 + readonly deliveryId: string;
731 + readonly prompt: string;
732 + readonly dispatched: Ref.Ref<boolean>;
733 + },
734 ) {
735 const reserved = yield* Ref.modify(activeRuns, (active) => {
736 if (active.has(task.id)) return [false, active] as const;
@@ -739,6 +764,12 @@ export const layer = Layer.effect(
764 }
765 return { task, threadId: null };
766 }
767 + if (
768 + webhook !== undefined &&
769 + (yield* sql`SELECT 1 FROM scheduled_task_webhook_pending WHERE delivery_id = ${webhook.deliveryId} AND state = 'completed'`)
770 + .length > 0
771 + )
772 + return { task: { ...active, lastRunStatus: "succeeded" as const }, threadId: null };
773 const parsedNextRunAt =
774 active.nextRunAt === null ? Option.none() : DateTime.make(active.nextRunAt);
775 if (
@@ -764,7 +795,22 @@ export const layer = Layer.effect(
795 if (reason !== null) return yield* new WebhookDeliverySkipped({ reason });
796 }
797
767 - yield* markRunning(active.id, startedAtIso);
798 + if (webhook === undefined) yield* markRunning(active.id, startedAtIso);
799 + else
800 + yield* sql
801 + .withTransaction(
802 + Effect.gen(function* () {
803 + yield* markRunning(active.id, startedAtIso, webhook.deliveryId);
804 + // Recovery must distinguish this delivery from other windows waiting on the same task.
805 + yield* sql`UPDATE scheduled_task_webhook_pending SET state = 'running' WHERE delivery_id = ${webhook.deliveryId}`;
806 + }),
807 + )
808 + .pipe(
809 + Effect.mapError((cause) =>
810 + taskError("Could not start webhook run.", { taskId: task.id, cause }),
811 + ),
812 + );
813 +
814 yield* notifyChanged;
815
816 const fireKey =
@@ -821,8 +867,9 @@ export const layer = Layer.effect(
867 .pipe(Effect.as(ThreadId.make(active.threadId))),
868 );
869
824 - const completedAt = yield* localNow;
870 const runSucceeded = result._tag === "Success";
871 + if (webhook !== undefined && runSucceeded) yield* Ref.set(webhook.dispatched, true);
872 + const completedAt = yield* localNow;
873 const lastRunStatus = runSucceeded ? ("succeeded" as const) : ("failed" as const);
874 const lastRunError = runSucceeded ? null : errorMessage(result.cause);
875 const threadId = runSucceeded ? result.value : null;
@@ -841,7 +888,7 @@ export const layer = Layer.effect(
888 // startedAtIso in the guard ensures this writes only to the row this
889 // run marked as running — a task deleted mid-run and recreated with
890 // the same id (idempotent commandId replay) must not be stamped.
844 - yield* markCompleted({
891 + const completion = markCompleted({
892 id: task.id,
893 completedAtIso: completed.updatedAt,
894 nextRunAtIso: completed.nextRunAt,
@@ -849,11 +896,32 @@ export const layer = Layer.effect(
896 error: lastRunError,
897 startedAtIso,
898 });
899 + if (webhook === undefined) yield* completion;
900 + else
901 + yield* sql
902 + .withTransaction(
903 + Effect.gen(function* () {
904 + const pending =
905 + yield* sql`SELECT 1 FROM scheduled_task_webhook_pending WHERE delivery_id = ${webhook.deliveryId}`;
906 + if (pending.length === 0) yield* completion;
907 + else {
908 + const counted =
909 + yield* sql`UPDATE scheduled_task_webhook_pending SET state = ${runSucceeded ? "completed" : "failed"}
910 + WHERE delivery_id = ${webhook.deliveryId} AND state NOT IN ('completed', 'failed') RETURNING delivery_id`;
911 + if (counted.length > 0) yield* completion;
912 + }
913 + }),
914 + )
915 + .pipe(
916 + Effect.mapError((cause) =>
917 + taskError("Could not record webhook completion.", { taskId: task.id, cause }),
918 + ),
919 + );
920 yield* notifyChanged;
921 }
922 return { task: completed, threadId };
923 }).pipe(
856 - Effect.onError((cause) => releaseStuckRun(task, errorMessage(cause))),
924 + Effect.onError((cause) => releaseStuckRun(task, errorMessage(cause), webhook?.deliveryId)),
925 Effect.ensuring(
926 Ref.update(activeRuns, (active) => {
927 const next = new Set(active);
@@ -930,7 +998,11 @@ export const layer = Layer.effect(
998 last_run_error = 'Run was interrupted by a server restart.',
999 next_run_at = ${nextRunAt(decoded.success, now)},
1000 updated_at = ${iso(now)},
933 - run_count = run_count + 1
1001 + run_count = run_count + CASE WHEN EXISTS (
1002 + SELECT 1 FROM scheduled_task_webhook_pending
1003 + WHERE task_id IS ${row.task_id} AND state = 'running'
1004 + AND delivery_id = scheduled_tasks.last_run_webhook_delivery_id
1005 + ) THEN 0 ELSE 1 END
1006 WHERE task_id IS ${row.task_id} AND last_run_status = 'running'
1007 `;
1008 return;
@@ -944,7 +1016,11 @@ export const layer = Layer.effect(
1016 SET last_run_status = 'failed',
1017 last_run_error = 'Run was interrupted by a server restart.',
1018 updated_at = ${iso(now)},
947 - run_count = run_count + 1
1019 + run_count = run_count + CASE WHEN EXISTS (
1020 + SELECT 1 FROM scheduled_task_webhook_pending
1021 + WHERE task_id IS ${row.task_id} AND state = 'running'
1022 + AND delivery_id = scheduled_tasks.last_run_webhook_delivery_id
1023 + ) THEN 0 ELSE 1 END
1024 WHERE task_id IS ${row.task_id} AND last_run_status = 'running'
1025 `;
1026 }),
@@ -1017,6 +1093,12 @@ export const layer = Layer.effect(
1093 prefix: input.schedule.signature.prefix,
1094 },
1095 maxDeliveryAgeMinutes: input.schedule.maxDeliveryAgeMinutes ?? null,
1096 + ...(input.schedule.dedupe === undefined
1097 + ? existingTask?.schedule.type === "webhook" &&
1098 + existingTask.schedule.dedupe !== undefined
1099 + ? { dedupe: existingTask.schedule.dedupe }
1100 + : {}
1101 + : { dedupe: input.schedule.dedupe }),
1102 ...webhookFilters(
1103 // Omitted keeps the stored filters, so a client that predates
1104 // them can re-save a task without clearing them.
@@ -1139,6 +1221,16 @@ export const layer = Layer.effect(
1221 next.delete(id);
1222 return next;
1223 }),
1224 + Ref.update(webhookRejectedRateWindows, (windows) => {
1225 + const next = new Map(windows);
1226 + next.delete(id);
1227 + return next;
1228 + }),
1229 + Ref.update(webhookDedupeLogWindows, (windows) => {
1230 + const next = new Map(windows);
1231 + next.delete(id);
1232 + return next;
1233 + }),
1234 Ref.update(webhookFilteredLogWindows, (windows) => {
1235 const next = new Map(windows);
1236 next.delete(id);
@@ -1243,6 +1335,7 @@ export const layer = Layer.effect(
1335 bodyBytes: row.body_bytes,
1336 outcome: row.outcome as ScheduledTaskWebhookDeliveryOutcome,
1337 signatureVerified: row.signature_verified === 1,
1338 + pendingWindow: row.outcome === "accepted" && row.pending_window === 1,
1339 missingFields: Option.getOrElse(decodeMissingFieldsJson(row.missing_fields_json), () => []),
1340 error: row.error,
1341 });
@@ -1251,15 +1344,17 @@ export const layer = Layer.effect(
1344 input,
1345 ) =>
1346 sql<WebhookDeliveryRow>`
1254 - SELECT * FROM scheduled_task_webhook_deliveries
1255 - WHERE task_id = ${input.id}
1347 + SELECT d.*, EXISTS (SELECT 1 FROM scheduled_task_webhook_pending p
1348 + WHERE p.delivery_id = d.delivery_id AND p.state IN ('pending', 'dispatching')) AS pending_window
1349 + FROM scheduled_task_webhook_deliveries d WHERE task_id = ${input.id}
1350 ORDER BY received_at DESC, rowid DESC
1351 `.pipe(
1352 Effect.map((rows) => ({
1259 - deliveries: (input.acceptAnyOutcome === true
1260 - ? rows
1261 - : rows.filter((row) => LEGACY_DELIVERY_OUTCOMES.has(row.outcome))
1262 - ).map(decodeDeliverySummary),
1353 + deliveries: rows
1354 + .filter(
1355 + (row) => input.acceptAnyOutcome === true || LEGACY_DELIVERY_OUTCOMES.has(row.outcome),
1356 + )
1357 + .map(decodeDeliverySummary),
1358 })),
1359 Effect.mapError((cause) =>
1360 taskError("Could not list webhook deliveries.", { taskId: input.id, cause }),
@@ -1270,15 +1365,19 @@ export const layer = Layer.effect(
1365 "ScheduledTaskService.getWebhookDelivery",
1366 )(function* (input) {
1367 const rows = yield* sql<WebhookDeliveryRow>`
1273 - SELECT * FROM scheduled_task_webhook_deliveries
1274 - WHERE task_id = ${input.id} AND delivery_id = ${input.deliveryId}
1368 + SELECT d.*, EXISTS (SELECT 1 FROM scheduled_task_webhook_pending p
1369 + WHERE p.delivery_id = d.delivery_id AND p.state IN ('pending', 'dispatching')) AS pending_window
1370 + FROM scheduled_task_webhook_deliveries d WHERE task_id = ${input.id} AND delivery_id = ${input.deliveryId}
1371 `.pipe(
1372 Effect.mapError((cause) =>
1373 taskError("Could not load webhook delivery.", { taskId: input.id, cause }),
1374 ),
1375 );
1376 const row = rows[0];
1281 - if (row === undefined) {
1377 + if (
1378 + row === undefined ||
1379 + (input.acceptAnyOutcome !== true && !LEGACY_DELIVERY_OUTCOMES.has(row.outcome))
1380 + ) {
1381 return yield* taskError("Webhook delivery not found.", { taskId: input.id });
1382 }
1383 return {
@@ -1293,6 +1392,24 @@ export const layer = Layer.effect(
1392 };
1393 });
1394
1395 + const deliveryRequestBody = (request: WebhookTriggerRequest) => {
1396 + const truncated = request.body.byteLength > WEBHOOK_DELIVERY_LOG_BODY_LIMIT;
1397 + return {
1398 + truncated,
1399 + body: truncated
1400 + ? new TextDecoder().decode(request.body.subarray(0, WEBHOOK_DELIVERY_LOG_BODY_LIMIT))
1401 + : request.bodyText,
1402 + };
1403 + };
1404 + const replaceDeliveryRequest = (deliveryId: string, request: WebhookTriggerRequest) => {
1405 + const logged = deliveryRequestBody(request);
1406 + return sql`UPDATE scheduled_task_webhook_deliveries SET
1407 + method = ${request.method}, query = ${redactQuery(request.query)},
1408 + headers_json = ${encodeHeadersJson(redactHeaders(request.headers))}, body = ${logged.body},
1409 + body_bytes = ${request.body.byteLength}, body_truncated = ${logged.truncated ? 1 : 0}
1410 + WHERE delivery_id = ${deliveryId}`.pipe(Effect.asVoid, Effect.mapError(dedupeError));
1411 + };
1412 +
1413 const recordDelivery = (input: {
1414 readonly id: string;
1415 readonly taskId: ScheduledTaskId;
@@ -1304,10 +1421,7 @@ export const layer = Layer.effect(
1421 readonly renderedPrompt: string | null;
1422 readonly error?: string;
1423 }) => {
1307 - const truncated = input.request.body.byteLength > WEBHOOK_DELIVERY_LOG_BODY_LIMIT;
1308 - const loggedBody = truncated
1309 - ? new TextDecoder().decode(input.request.body.subarray(0, WEBHOOK_DELIVERY_LOG_BODY_LIMIT))
1310 - : input.request.bodyText;
1424 + const { truncated, body: loggedBody } = deliveryRequestBody(input.request);
1425 return sql
1426 .withTransaction(
1427 Effect.gen(function* () {
@@ -1338,17 +1452,28 @@ export const layer = Layer.effect(
1452 rendered_prompt = excluded.rendered_prompt,
1453 error = excluded.error
1454 `;
1341 - const filtered = input.outcome === "filtered" ? 1 : 0;
1455 + // Each suppression bucket retains its own rows, never real run history.
1456 + const bucket = (outcome: string, error: string | undefined) =>
1457 + outcome === "filtered"
1458 + ? "filtered"
1459 + : outcome === "duplicate" ||
1460 + outcome === "coalesced" ||
1461 + outcome === "rate_limited" ||
1462 + error === PROMPT_TOO_LONG
1463 + ? "dedupe"
1464 + : "main";
1465 + const retentionBucket = bucket(input.outcome, input.error);
1466 + const rowBucket = sql.literal(`CASE WHEN outcome = 'filtered' THEN 'filtered'
1467 + WHEN outcome IN ('duplicate', 'coalesced', 'rate_limited') OR error = 'The filled-in prompt is too long.' THEN 'dedupe'
1468 + ELSE 'main' END`);
1469 yield* sql`
1470 DELETE FROM scheduled_task_webhook_deliveries
1344 - WHERE task_id = ${input.taskId}
1345 - AND (outcome = 'filtered') = ${filtered}
1471 + WHERE task_id = ${input.taskId} AND ${rowBucket} = ${retentionBucket}
1472 AND delivery_id NOT IN (
1473 SELECT delivery_id FROM scheduled_task_webhook_deliveries
1348 - WHERE task_id = ${input.taskId} AND (outcome = 'filtered') = ${filtered}
1349 - -- rowid breaks timestamp ties in arrival order; delivery ids are random.
1474 + WHERE task_id = ${input.taskId} AND ${rowBucket} = ${retentionBucket}
1475 ORDER BY received_at DESC, rowid DESC
1351 - LIMIT ${filtered ? WEBHOOK_FILTERED_DELIVERY_RETENTION : WEBHOOK_DELIVERY_RETENTION}
1476 + LIMIT ${retentionBucket === "main" ? WEBHOOK_DELIVERY_RETENTION : retentionBucket === "filtered" ? WEBHOOK_FILTERED_DELIVERY_RETENTION : WEBHOOK_DEDUPE_DELIVERY_RETENTION}
1477 )
1478 `;
1479 }),
@@ -1367,13 +1492,15 @@ export const layer = Layer.effect(
1492 WHERE delivery_id = ${deliveryId}
1493 `.pipe(Effect.ignore);
1494
1370 - /**
1371 - * Sliding one-minute window counting every request with a valid token,
1372 - * including ones the signature check later rejects.
1373 - */
1374 - const takeRateSlot = (id: ScheduledTaskId, nowMs: number) =>
1495 + /** Sliding one-minute window for deliveries that can start work. */
1496 + const takeRateSlot = (
1497 + id: ScheduledTaskId,
1498 + nowMs: number,
1499 + rateWindows = webhookRateWindows,
1500 + reservation = { at: nowMs },
1501 + ) =>
1502 Ref.modify(
1376 - webhookRateWindows,
1503 + rateWindows,
1504 (
1505 windows,
1506 ): readonly [
@@ -1381,7 +1508,7 @@ export const layer = Layer.effect(
1508 ReadonlyMap<ScheduledTaskId, RateWindow>,
1509 ] => {
1510 const current = windows.get(id);
1384 - const recent = (current?.accepted ?? []).filter((at) => nowMs - at < 60_000);
1511 + const recent = (current?.accepted ?? []).filter((entry) => nowMs - entry.at < 60_000);
1512 if (recent.length >= WEBHOOK_RATE_LIMIT_PER_MINUTE) {
1513 const first = current?.rejectedLogged !== true;
1514 return [
@@ -1391,11 +1518,23 @@ export const layer = Layer.effect(
1518 }
1519 return [
1520 "allowed",
1394 - new Map(windows).set(id, { accepted: [...recent, nowMs], rejectedLogged: false }),
1521 + new Map(windows).set(id, { accepted: [...recent, reservation], rejectedLogged: false }),
1522 ];
1523 },
1524 );
1525
1526 + const takeDedupeLogSlot = (id: ScheduledTaskId, nowMs: number) =>
1527 + Ref.modify(webhookDedupeLogWindows, (windows) => {
1528 + const recent = (windows.get(id) ?? []).filter((at) => nowMs - at < 60_000);
1529 + return recent.length >= WEBHOOK_DEDUPE_LOG_PER_MINUTE
1530 + ? ([false, windows] as const)
1531 + : ([true, new Map(windows).set(id, [...recent, nowMs])] as const);
1532 + });
1533 + const releaseReplay = (deliveryId: string) =>
1534 + releaseWebhookReplay(deliveryId).pipe(
1535 + Effect.provideService(SqlClient.SqlClient, sql),
1536 + Effect.mapError(dedupeError),
1537 + );
1538 /** Sliding one-minute window of filtered requests written to the delivery log. */
1539 const takeFilteredLogSlot = (id: ScheduledTaskId, nowMs: number) =>
1540 Ref.modify(
@@ -1424,6 +1563,83 @@ export const layer = Layer.effect(
1563
1564 const serviceScope = yield* Effect.scope;
1565
1566 + const releaseFailedDelivery = Effect.fn("ScheduledTaskService.releaseFailedDelivery")(
1567 + function* (deliveryId: string) {
1568 + yield* releaseReplay(deliveryId);
1569 + yield* sql`DELETE FROM scheduled_task_webhook_relay_deliveries WHERE relay_delivery_id IN (
1570 + SELECT substr(value, ${"delivery:relay:".length + 1}) FROM scheduled_task_webhook_pending, json_each(delivery_ids_json)
1571 + WHERE delivery_id = ${deliveryId} AND value LIKE 'delivery:relay:%'
1572 + )`.pipe(Effect.mapError(dedupeError));
1573 + if (deliveryId.startsWith("delivery:relay:"))
1574 + yield* sql`DELETE FROM scheduled_task_webhook_relay_deliveries WHERE relay_delivery_id = ${deliveryId.slice("delivery:relay:".length)}`.pipe(
1575 + Effect.mapError(dedupeError),
1576 + );
1577 + },
1578 + (effect) => sql.withTransaction(effect).pipe(Effect.mapError(dedupeError)),
1579 + );
1580 +
1581 + const runWebhookDelivery = Effect.fn("ScheduledTaskService.runWebhookDelivery")(function* (
1582 + task: ScheduledTask,
1583 + deliveryId: string,
1584 + prompt: string,
1585 + ) {
1586 + const runOutcome = (outcome: "started" | "skipped" | "failed") =>
1587 + Effect.all([
1588 + Effect.annotateCurrentSpan({ "scheduled_task.webhook.run_outcome": outcome }),
1589 + Metrics.increment(Metrics.webhookRunsTotal, { outcome }),
1590 + ]);
1591 + const dispatched = yield* Ref.make(false);
1592 + return yield* runTask(task, "webhook", { deliveryId, prompt, dispatched }).pipe(
1593 + Effect.flatMap(({ task: completed }) =>
1594 + completed.lastRunStatus === "failed"
1595 + ? runOutcome("failed").pipe(
1596 + Effect.andThen(markDeliveryFailed(deliveryId, "The run failed to start.")),
1597 + Effect.andThen(releaseFailedDelivery(deliveryId)),
1598 + )
1599 + : runOutcome("started"),
1600 + ),
1601 + Effect.catchTags({
1602 + WebhookDeliverySkipped: (skipped) =>
1603 + runOutcome("skipped").pipe(
1604 + Effect.andThen(markDeliveryFailed(deliveryId, skipped.reason)),
1605 + Effect.andThen(releaseFailedDelivery(deliveryId)),
1606 + ),
1607 + }),
1608 + Effect.as(true),
1609 + // Dispatch success owns the replay key even if later bookkeeping fails.
1610 + Effect.catchCause((cause) =>
1611 + Effect.gen(function* () {
1612 + const started = yield* Ref.get(dispatched);
1613 + yield* Effect.logWarning("Webhook dispatch failed", { taskId: task.id, cause });
1614 + yield* runOutcome(started ? "started" : "failed");
1615 + yield* markDeliveryFailed(
1616 + deliveryId,
1617 + started
1618 + ? "The run started, but completion could not be recorded."
1619 + : "The run failed to start.",
1620 + );
1621 + if (!started) {
1622 + yield* sql
1623 + .withTransaction(
1624 + Effect.gen(function* () {
1625 + const counted =
1626 + yield* sql`UPDATE scheduled_task_webhook_pending SET state = 'failed'
1627 + WHERE delivery_id = ${deliveryId} AND state = 'running' RETURNING delivery_id`;
1628 + if (counted.length > 0)
1629 + yield* sql`UPDATE scheduled_tasks SET run_count = run_count + 1
1630 + WHERE task_id = ${task.id} AND created_at = ${task.createdAt}`;
1631 + }),
1632 + )
1633 + .pipe(Effect.mapError(dedupeError));
1634 + yield* releaseFailedDelivery(deliveryId);
1635 + }
1636 + // Pending windows retry the same command id to finish bookkeeping.
1637 + return !started;
1638 + }),
1639 + ),
1640 + );
1641 + });
1642 +
1643 // Delivery attributes belong to the enclosing triggerWebhook span.
1644 const observeDelivery = Effect.fnUntraced(function* (
1645 request: WebhookTriggerRequest,
@@ -1571,71 +1787,168 @@ export const layer = Layer.effect(
1787 ...(details.error === undefined ? {} : { error: details.error }),
1788 });
1789
1790 + // Reject floods before signature work or rejection-log writes; valid
1791 + // deliveries give this reservation back once validation succeeds.
1792 + const rejectionReservation = { at: DateTime.toEpochMillis(now) };
1793 + const rejectionSlot = yield* takeRateSlot(
1794 + task.id,
1795 + DateTime.toEpochMillis(now),
1796 + webhookRejectedRateWindows,
1797 + rejectionReservation,
1798 + );
1799 + if (rejectionSlot !== "allowed") {
1800 + if (rejectionSlot === "first_rejected") yield* log("rate_limited");
1801 + yield* observe("rate_limited");
1802 + return yield* releaseClaim({ _tag: "rate_limited", outcome: "rate_limited" } as const);
1803 + }
1804 + if (!task.enabled) {
1805 + yield* log("disabled");
1806 + yield* observe("disabled");
1807 + return { _tag: "disabled" as const };
1808 + }
1809 const signature = schedule.signature;
1575 - const signatureVerified =
1576 - signature === null ||
1577 - (row.webhook_secret !== null &&
1810 + if (signature !== null) {
1811 + const verified =
1812 + row.webhook_secret !== null &&
1813 verifyWebhookSignature({
1814 signature,
1815 secret: row.webhook_secret,
1816 headers: request.headers,
1817 body: request.body,
1583 - }));
1818 + });
1819 + if (!verified) {
1820 + yield* log("rejected_signature");
1821 + yield* observe("rejected_signature");
1822 + return { _tag: "rejected_signature" as const };
1823 + }
1824 + }
1825 const maxAgeMinutes = schedule.maxDeliveryAgeMinutes ?? null;
1585 - const tooOld =
1826 + if (
1827 maxAgeMinutes !== null &&
1828 DateTime.toEpochMillis(now) - DateTime.toEpochMillis(receivedAt) >
1588 - maxAgeMinutes * 60_000;
1589 -
1590 - // Filters: checked after the signature and age, and before the rate
1591 - // limit, so requests the task ignores never take a run's slot. The
1592 - // request is accepted so the sender doesn't retry it, and its relay
1593 - // claim is released because nothing ran.
1594 - if (task.enabled && signatureVerified && !tooOld) {
1595 - const failedFilter = findFailedWebhookFilter(schedule.filters ?? [], request);
1596 - if (failedFilter !== null) {
1597 - if (yield* takeFilteredLogSlot(task.id, DateTime.toEpochMillis(now))) {
1598 - yield* log("filtered", {
1599 - signatureVerified: signature !== null,
1600 - error: describeFailedWebhookFilter(failedFilter),
1601 - });
1602 - }
1603 - yield* observe("filtered");
1604 - return yield* releaseClaim({
1605 - _tag: "accepted" as const,
1606 - deliveryId,
1607 - outcome: "filtered" as const,
1608 - });
1609 - }
1829 + maxAgeMinutes * 60_000
1830 + ) {
1831 + yield* log("expired", { signatureVerified: signature !== null });
1832 + yield* observe("expired");
1833 + return { _tag: "expired" as const };
1834 }
1835
1612 - // Only the first rejected request in a window is logged, so a flood
1613 - // cannot write rows or push the real deliveries out of the log.
1614 - const slot = yield* takeRateSlot(task.id, DateTime.toEpochMillis(now));
1615 - if (slot !== "allowed") {
1616 - if (slot === "first_rejected") yield* log("rate_limited");
1617 - yield* observe("rate_limited");
1836 + const refundRejectionSlot = Ref.update(webhookRejectedRateWindows, (windows) => {
1837 + const current = windows.get(task.id);
1838 + if (current === undefined) return windows;
1839 + return new Map(windows).set(task.id, {
1840 + ...current,
1841 + accepted: current.accepted.filter((entry) => entry !== rejectionReservation),
1842 + });
1843 + });
1844 +
1845 + const failedFilter = findFailedWebhookFilter(schedule.filters ?? [], request);
1846 + if (failedFilter !== null) {
1847 + if (yield* takeFilteredLogSlot(task.id, DateTime.toEpochMillis(now))) {
1848 + yield* log("filtered", {
1849 + signatureVerified: signature !== null,
1850 + error: describeFailedWebhookFilter(failedFilter),
1851 + });
1852 + }
1853 + yield* refundRejectionSlot;
1854 + yield* observe("filtered");
1855 return yield* releaseClaim({
1619 - _tag: "rate_limited" as const,
1620 - outcome: "rate_limited" as const,
1856 + _tag: "accepted" as const,
1857 + deliveryId,
1858 + outcome: "filtered" as const,
1859 });
1860 }
1623 - if (!task.enabled) {
1624 - yield* log("disabled");
1625 - yield* observe("disabled");
1626 - return { _tag: "disabled" as const };
1861 +
1862 + const runWindow = (yield* Ref.get(webhookRateWindows)).get(task.id);
1863 + if (
1864 + schedule.dedupe === undefined &&
1865 + (runWindow?.accepted.filter((entry) => DateTime.toEpochMillis(now) - entry.at < 60_000)
1866 + .length ?? 0) >= WEBHOOK_RATE_LIMIT_PER_MINUTE
1867 + ) {
1868 + if ((yield* takeRateSlot(task.id, DateTime.toEpochMillis(now))) === "first_rejected")
1869 + yield* log("rate_limited");
1870 + yield* observe("rate_limited");
1871 + return yield* releaseClaim({ _tag: "rate_limited", outcome: "rate_limited" } as const);
1872 }
1628 - if (!signatureVerified) {
1629 - yield* log("rejected_signature");
1630 - yield* observe("rejected_signature");
1631 - return { _tag: "rejected_signature" as const };
1873 + const reserve = takeRateSlot(task.id, DateTime.toEpochMillis(now)).pipe(
1874 + Effect.flatMap((slot) => (slot === "allowed" ? Effect.void : Effect.fail(slot))),
1875 + );
1876 + const queueKey = `${task.id}\u0000${task.createdAt}`;
1877 + const checkCapacity = Effect.gen(function* () {
1878 + const [row] = yield* sql<{
1879 + count: number;
1880 + }>`SELECT COUNT(*) AS count FROM scheduled_task_webhook_pending
1881 + WHERE task_id = ${task.id} AND state IN ('pending', 'dispatching', 'running')`;
1882 + if (
1883 + (row?.count ?? 0) + ((yield* Ref.get(webhookQueued)).get(queueKey) ?? 0) >=
1884 + WEBHOOK_MAX_QUEUED_PER_TASK
1885 + )
1886 + return yield* Effect.fail("queue_full" as const);
1887 + }).pipe(
1888 + Effect.mapError((cause) => (cause === "queue_full" ? cause : dedupeError(cause))),
1889 + );
1890 + const admit = <A, E>(effect: Effect.Effect<A, E>) =>
1891 + sql.withTransaction(effect).pipe(
1892 + Effect.catch((cause) =>
1893 + cause === "first_rejected" || cause === "rejected" || cause === "queue_full"
1894 + ? Effect.succeed({
1895 + outcome:
1896 + cause === "queue_full"
1897 + ? ("queue_full" as const)
1898 + : ("rate_limited" as const),
1899 + log: cause === "first_rejected",
1900 + })
1901 + : Effect.fail(isScheduledTaskError(cause) ? cause : dedupeError(cause)),
1902 + ),
1903 + );
1904 + const deduped =
1905 + schedule.dedupe === undefined
1906 + ? "run"
1907 + : yield* admit(
1908 + Effect.gen(function* () {
1909 + const outcome = yield* acceptWebhookDedupe(
1910 + task,
1911 + request,
1912 + deliveryId,
1913 + DateTime.toEpochMillis(now),
1914 + reserve,
1915 + takeDedupeLogSlot(task.id, DateTime.toEpochMillis(now)),
1916 + checkCapacity,
1917 + (ownerId) => replaceDeliveryRequest(ownerId, request),
1918 + ).pipe(Effect.provideService(SqlClient.SqlClient, sql));
1919 + if (
1920 + outcome === "pending" ||
1921 + outcome === "coalesced" ||
1922 + (outcome === "duplicate" &&
1923 + (yield* takeDedupeLogSlot(task.id, DateTime.toEpochMillis(now))))
1924 + )
1925 + yield* log(outcome === "pending" ? "accepted" : outcome, {
1926 + signatureVerified: signature !== null,
1927 + });
1928 + return outcome;
1929 + }),
1930 + );
1931 + if (typeof deduped !== "string") {
1932 + yield* refundRejectionSlot;
1933 + if (deduped.log) yield* log("rate_limited");
1934 + yield* observe(deduped.outcome);
1935 + return yield* releaseClaim({ _tag: "rate_limited", outcome: deduped.outcome } as const);
1936 }
1633 - if (tooOld) {
1634 - yield* log("expired", { signatureVerified: signature !== null });
1635 - yield* observe("expired");
1636 - return { _tag: "expired" as const };
1937 + if (deduped !== "run") {
1938 + if (deduped !== "coalesced" && deduped !== "coalesced_unlogged")
1939 + yield* refundRejectionSlot;
1940 + const outcome =
1941 + deduped === "pending"
1942 + ? "accepted"
1943 + : deduped === "coalesced_unlogged"
1944 + ? "coalesced"
1945 + : deduped;
1946 + yield* observe(outcome);
1947 + if (deduped === "duplicate")
1948 + return yield* releaseClaim({ _tag: "accepted", deliveryId, outcome } as const);
1949 + claimAcquired = false;
1950 + return { _tag: "accepted", deliveryId, outcome } as const;
1951 }
1638 -
1952 const frameId = yield* crypto.randomBytes(6).pipe(
1953 Effect.map((bytes) => Buffer.from(bytes).toString("hex")),
1954 Effect.mapError((cause) => taskError("Could not generate frame id.", { cause })),
@@ -1648,85 +1961,192 @@ export const layer = Layer.effect(
1961 signatureVerified: signature !== null,
1962 missing: rendered.missing,
1963 renderedPrompt: rendered.prompt,
1651 - error: "The filled-in prompt is too long.",
1964 + error: PROMPT_TOO_LONG,
1965 });
1966 + yield* releaseReplay(deliveryId);
1967 yield* observe("prompt_too_long");
1654 - return { _tag: "accepted" as const, deliveryId, outcome: "prompt_too_long" as const };
1968 + return yield* releaseClaim({
1969 + _tag: "accepted" as const,
1970 + deliveryId,
1971 + outcome: "prompt_too_long" as const,
1972 + });
1973 }
1974 // Bound the deliveries one task holds, so steady traffic to a stuck
1975 // task cannot pile up parked fibers.
1658 - const queueKey = `${task.id}\u0000${task.createdAt}`;
1659 - const queued = yield* Ref.modify(webhookQueued, (counts) => {
1660 - const count = counts.get(queueKey) ?? 0;
1661 - return count >= WEBHOOK_MAX_QUEUED_PER_TASK
1662 - ? ([false, counts] as const)
1663 - : ([true, new Map(counts).set(queueKey, count + 1)] as const);
1976 + yield* refundRejectionSlot;
1977 + let queueReserved = false;
1978 + const release = Effect.suspend(() => {
1979 + if (!queueReserved) return Effect.void;
1980 + queueReserved = false;
1981 + return Ref.update(webhookQueued, (counts) => {
1982 + const next = new Map(counts);
1983 + const count = (next.get(queueKey) ?? 1) - 1;
1984 + if (count <= 0) next.delete(queueKey);
1985 + else next.set(queueKey, count);
1986 + return next;
1987 + });
1988 });
1665 - if (!queued) {
1666 - yield* observe("queue_full");
1989 + // Serialize immediate reservations with persisted window admission.
1990 + const reserveImmediate = Effect.gen(function* () {
1991 + yield* checkCapacity;
1992 + yield* reserve;
1993 + yield* Ref.update(webhookQueued, (counts) =>
1994 + new Map(counts).set(queueKey, (counts.get(queueKey) ?? 0) + 1),
1995 + );
1996 + queueReserved = true;
1997 + });
1998 + const admitted = yield* admit(
1999 + Effect.gen(function* () {
2000 + const outcome = yield* acceptImmediateWebhookDedupe(
2001 + task,
2002 + request,
2003 + deliveryId,
2004 + DateTime.toEpochMillis(now),
2005 + reserveImmediate,
2006 + ).pipe(Effect.provideService(SqlClient.SqlClient, sql));
2007 + if (outcome === "duplicate") {
2008 + if (yield* takeDedupeLogSlot(task.id, DateTime.toEpochMillis(now)))
2009 + yield* log("duplicate", { signatureVerified: signature !== null });
2010 + } else
2011 + yield* log("accepted", {
2012 + signatureVerified: signature !== null,
2013 + missing: rendered.missing,
2014 + renderedPrompt: rendered.prompt,
2015 + });
2016 + return outcome;
2017 + }),
2018 + ).pipe(Effect.onError(() => release));
2019 + if (typeof admitted !== "string") {
2020 + yield* release;
2021 + if (admitted.log) yield* log("rate_limited");
2022 + yield* observe(admitted.outcome);
2023 return yield* releaseClaim({
1668 - _tag: "rate_limited" as const,
1669 - outcome: "queue_full" as const,
1670 - });
2024 + _tag: "rate_limited",
2025 + outcome: admitted.outcome,
2026 + } as const);
2027 + }
2028 + if (admitted === "duplicate") {
2029 + yield* release;
2030 + yield* observe("duplicate");
2031 + return yield* releaseClaim({
2032 + _tag: "accepted",
2033 + deliveryId,
2034 + outcome: "duplicate",
2035 + } as const);
2036 }
1672 - const release = Ref.update(webhookQueued, (counts) => {
1673 - const next = new Map(counts);
1674 - const count = (next.get(queueKey) ?? 1) - 1;
1675 - if (count <= 0) next.delete(queueKey);
1676 - else next.set(queueKey, count);
1677 - return next;
1678 - });
1679 - yield* log("accepted", {
1680 - signatureVerified: signature !== null,
1681 - missing: rendered.missing,
1682 - renderedPrompt: rendered.prompt,
1683 - }).pipe(Effect.onError(() => release));
2037 yield* observe("accepted");
2038 const permit = yield* webhookPermit(task.id);
1686 - const runOutcome = (outcome: "started" | "skipped" | "failed") =>
1687 - Effect.all([
1688 - Effect.annotateCurrentSpan({ "scheduled_task.webhook.run_outcome": outcome }),
1689 - Metrics.increment(Metrics.webhookRunsTotal, { outcome }),
1690 - ]);
1691 - yield* runTask(task, "webhook", { deliveryId, prompt: rendered.prompt }).pipe(
1692 - Effect.flatMap(({ task: completed }) =>
1693 - completed.lastRunStatus === "failed"
1694 - ? runOutcome("failed").pipe(
1695 - Effect.andThen(markDeliveryFailed(deliveryId, "The run failed to start.")),
1696 - )
1697 - : runOutcome("started"),
1698 - ),
1699 - Effect.catchTags({
1700 - WebhookDeliverySkipped: (skipped) =>
1701 - runOutcome("skipped").pipe(
1702 - Effect.andThen(markDeliveryFailed(deliveryId, skipped.reason)),
1703 - ),
1704 - }),
1705 - // The log is readable over RPC, so it gets a fixed reason; the
1706 - // cause, which can carry request data, stays in the server log.
1707 - Effect.catchCause((cause) =>
1708 - Effect.logWarning("Webhook dispatch failed", { taskId: task.id, cause }).pipe(
1709 - Effect.andThen(runOutcome("failed")),
1710 - Effect.andThen(markDeliveryFailed(deliveryId, "The run failed to start.")),
1711 - ),
1712 - ),
2039 + yield* runWebhookDelivery(task, deliveryId, rendered.prompt).pipe(
2040 permit.withPermits(1),
1714 - Effect.withSpan("ScheduledTaskService.runWebhookDelivery", {
1715 - root: true,
1716 - attributes: {
1717 - "scheduled_task.id": task.id,
1718 - "scheduled_task.webhook.delivery_id": deliveryId,
1719 - },
1720 - }),
2041 Effect.ensuring(release),
2042 Effect.forkIn(serviceScope),
2043 );
2044 // Dispatch owns the accepted delivery now; retries must keep its claim.
2045 claimAcquired = false;
2046 return { _tag: "accepted" as const, deliveryId, outcome: "accepted" as const };
1727 - }).pipe(Effect.onError(() => releaseClaim(undefined))),
2047 + }).pipe(
2048 + Effect.onError(() =>
2049 + Effect.all([
2050 + releaseClaim(undefined),
2051 + (request.relayDeliveryId === undefined || claimAcquired
2052 + ? releaseReplay(deliveryId)
2053 + : Effect.void
2054 + ).pipe(Effect.ignore),
2055 + ]),
2056 + ),
2057 + ),
2058 + );
2059 + });
2060 +
2061 + const dispatchPending = Effect.fn("ScheduledTaskService.dispatchPendingWebhooks")(function* () {
2062 + const now = DateTime.toEpochMillis(yield* localNow);
2063 + yield* sql`UPDATE scheduled_task_webhook_pending SET state = 'dispatching' WHERE state = 'pending' AND due_at <= ${now}`;
2064 + const tasks = yield* sql<{ task_id: ScheduledTaskId }>`
2065 + SELECT DISTINCT task_id FROM scheduled_task_webhook_pending WHERE state IN ('dispatching', 'running', 'completed', 'failed')
2066 + `;
2067 + yield* Effect.forEach(
2068 + tasks,
2069 + ({ task_id: taskId }) =>
2070 + Effect.gen(function* () {
2071 + const reserved = yield* Ref.modify(pendingInFlight, (active) =>
2072 + active.has(taskId)
2073 + ? ([false, active] as const)
2074 + : ([true, new Set(active).add(taskId)] as const),
2075 + );
2076 + if (!reserved) return;
2077 + const permit = yield* webhookPermit(taskId);
2078 + // One worker per task waits on its permit, then drains persisted rows in order.
2079 + // Other tasks remain independent and ticks never reload an active task's payloads.
2080 + const dispatch = Effect.gen(function* () {
2081 + while (true) {
2082 + const [entry] = yield* sql<{
2083 + delivery_id: string;
2084 + task_json: string;
2085 + request_json: string;
2086 + state: string;
2087 + }>`
2088 + SELECT delivery_id, task_json, request_json, state FROM scheduled_task_webhook_pending
2089 + WHERE task_id = ${taskId} AND state IN ('dispatching', 'running', 'completed', 'failed') ORDER BY due_at, rowid LIMIT 1
2090 + `;
2091 + if (entry === undefined) return;
2092 + const remove = sql`DELETE FROM scheduled_task_webhook_pending WHERE delivery_id = ${entry.delivery_id}`;
2093 + if (entry.state === "completed" || entry.state === "failed") {
2094 + if (entry.state === "failed") {
2095 + yield* markDeliveryFailed(entry.delivery_id, "The run failed to start.");
2096 + yield* releaseFailedDelivery(entry.delivery_id);
2097 + }
2098 + yield* remove;
2099 + continue;
2100 + }
2101 + const prepared = yield* Effect.gen(function* () {
2102 + const task = yield* decodePendingTask(entry.task_json);
2103 + const request = yield* decodePendingRequest(entry.request_json);
2104 + const frameId = Buffer.from(yield* crypto.randomBytes(6)).toString("hex");
2105 + return { task, rendered: renderWebhookPrompt(task.prompt, request, frameId) };
2106 + }).pipe(Effect.exit);
2107 + if (Exit.isFailure(prepared)) {
2108 + if (Cause.hasInterruptsOnly(prepared.cause))
2109 + return yield* Effect.failCause(prepared.cause);
2110 + yield* markDeliveryFailed(
2111 + entry.delivery_id,
2112 + "The pending webhook payload could not be prepared.",
2113 + );
2114 + yield* releaseFailedDelivery(entry.delivery_id);
2115 + yield* remove;
2116 + continue;
2117 + }
2118 + const { task, rendered } = prepared.value;
2119 + if (rendered.prompt.trim().length > PROVIDER_SEND_TURN_MAX_INPUT_CHARS) {
2120 + yield* markDeliveryFailed(entry.delivery_id, PROMPT_TOO_LONG);
2121 + yield* releaseFailedDelivery(entry.delivery_id);
2122 + } else {
2123 + yield* sql`UPDATE scheduled_task_webhook_deliveries SET outcome = 'accepted', error = NULL, rendered_prompt = ${rendered.prompt.slice(0, WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT)}, missing_fields_json = ${encodeMissingFieldsJson(rendered.missing)} WHERE delivery_id = ${entry.delivery_id}`;
2124 + if (!(yield* runWebhookDelivery(task, entry.delivery_id, rendered.prompt)))
2125 + return;
2126 + }
2127 + yield* remove;
2128 + }
2129 + });
2130 + yield* dispatch.pipe(
2131 + permit.withPermits(1),
2132 + Effect.catch((cause) =>
2133 + Effect.logWarning("Pending webhook dispatch failed", { cause }),
2134 + ),
2135 + Effect.ensuring(
2136 + Ref.update(pendingInFlight, (active) => {
2137 + const next = new Set(active);
2138 + next.delete(taskId);
2139 + return next;
2140 + }),
2141 + ),
2142 + Effect.withSpan("ScheduledTaskService.dispatchPendingWebhook", { root: true }),
2143 + Effect.forkIn(serviceScope),
2144 + );
2145 + }),
2146 + { discard: true },
2147 );
2148 });
2149 + yield* scheduler.register("webhook-coalescing", dispatchPending());
2150
2151 return ScheduledTaskService.of({
2152 list,
apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts
+1903 -125
@@ -3,8 +3,13 @@ import * as NodeCrypto from "node:crypto";
3
4 import * as NodePlatformCrypto from "@effect/platform-node/NodeCrypto";
5 import { assert, it } from "@effect/vitest";
6 -import { ScheduledTaskUpsertInput, SecretRequestError } from "@t3tools/contracts";
6 +import {
7 + ScheduledTaskUpsertInput,
8 + ScheduledTaskWebhookDeliveryId,
9 + SecretRequestError,
10 +} from "@t3tools/contracts";
11 import * as DateTime from "effect/DateTime";
12 +import * as Crypto from "effect/Crypto";
13 import * as Deferred from "effect/Deferred";
14 import * as Effect from "effect/Effect";
15 import * as Fiber from "effect/Fiber";
@@ -24,9 +29,94 @@ import * as ManagedProjectFolders from "../project/ManagedProjectFolders.ts";
29 import * as SqlitePersistence from "../persistence/Sqlite.ts";
30 import * as Scheduler from "../scheduling/Scheduler.ts";
31 import * as ScheduledTaskService from "./ScheduledTaskService.ts";
32 +import { renderWebhookPrompt } from "./webhookTemplate.ts";
33
34 const decodeUpsertInput = Schema.decodeUnknownEffect(ScheduledTaskUpsertInput);
35
36 +it.effect(
37 + "preserves omitted dedupe settings, releases duplicate claims, and retains coalesced claims",
38 + () =>
39 + withService(({ service, sql, launches }) =>
40 + Effect.gen(function* () {
41 + const dedupe = {
42 + replayKey: "{{headers.delivery-id}}",
43 + coalesceKey: "{{body.group}}",
44 + coalesceWindowSeconds: 10,
45 + };
46 + const { task } = yield* service.upsert(
47 + yield* webhookTaskInput({ schedule: { type: "webhook", dedupe } }),
48 + );
49 + const updated = yield* service.upsert(yield* webhookTaskInput());
50 + assert.deepEqual(
51 + updated.task.schedule.type === "webhook" && updated.task.schedule.dedupe,
52 + dedupe,
53 + );
54 + const request = (id: string, bodyText = "{}", relayDeliveryId?: string) =>
55 + requestFor(task, {
56 + headers: { "delivery-id": id },
57 + bodyText,
58 + body: new TextEncoder().encode(bodyText),
59 + ...(relayDeliveryId === undefined ? {} : { relayDeliveryId }),
60 + });
61 + yield* service.triggerWebhook(request("A"));
62 + yield* Queue.take(launches);
63 + yield* service.triggerWebhook(request("B1", '{"group":"B"}'));
64 + for (let i = 0; i < 5; i++) {
65 + const duplicate = yield* service.triggerWebhook(request("A", "{}", `duplicate-${i}`));
66 + assert.equal(duplicate._tag === "accepted" && duplicate.outcome, "duplicate");
67 + const coalesced = yield* service.triggerWebhook(
68 + request(`B${i + 2}`, '{"group":"B"}', `coalesced-${i}`),
69 + );
70 + assert.equal(coalesced._tag === "accepted" && coalesced.outcome, "coalesced");
71 + }
72 + const claims = yield* sql<{
73 + relay_delivery_id: string;
74 + }>`SELECT relay_delivery_id FROM scheduled_task_webhook_relay_deliveries ORDER BY relay_delivery_id`;
75 + assert.deepEqual(
76 + claims.map((claim) => claim.relay_delivery_id),
77 + Array.from({ length: 5 }, (_, i) => `coalesced-${i}`),
78 + );
79 + assert.isTrue(
80 + (yield* service.listWebhookDeliveries({ id: task.id })).deliveries.every(
81 + (delivery) => delivery.outcome !== "duplicate" && delivery.outcome !== "coalesced",
82 + ),
83 + );
84 + const fresh = yield* service.triggerWebhook(request("C", "{}", "duplicate-0"));
85 + assert.equal(fresh._tag === "accepted" && fresh.outcome, "accepted");
86 + yield* Queue.take(launches);
87 + const cleared = yield* service.upsert(
88 + yield* webhookTaskInput({ schedule: { type: "webhook", dedupe: {} } }),
89 + );
90 + assert.deepEqual(
91 + cleared.task.schedule.type === "webhook" && cleared.task.schedule.dedupe,
92 + {},
93 + );
94 + }),
95 + ),
96 +);
97 +
98 +const waitForSpan = <A, E, R>(work: Effect.Effect<A, E, R>, name: string) =>
99 + Effect.gen(function* () {
100 + const done = yield* Deferred.make<void>();
101 + const tracer = Tracer.make({
102 + span: (options) => {
103 + const span = new Tracer.NativeSpan(options);
104 + const end = span.end.bind(span);
105 + span.end = (...args) => {
106 + end(...args);
107 + if (span.name === name) Deferred.doneUnsafe(done, Effect.void);
108 + };
109 + return span;
110 + },
111 + });
112 + const result = yield* work.pipe(Effect.withTracer(tracer));
113 + yield* Deferred.await(done);
114 + return result;
115 + });
116 +
117 +const drainPendingDispatch = (workers: Map<string, Effect.Effect<void>>) =>
118 + waitForSpan(workers.get("webhook-coalescing")!, "ScheduledTaskService.dispatchPendingWebhook");
119 +
120 type LaunchInput = ThreadLaunchService.ThreadLaunchInput;
121
122 const webhookTaskInput = (overrides: Record<string, unknown> = {}) =>
@@ -70,20 +160,76 @@ const withService = <A, E>(
160 readonly sql: SqlClient.SqlClient;
161 readonly secretsByRef: Map<string, string>;
162 }) => Effect.Effect<A, E, never>,
73 - options: { readonly gate?: Deferred.Deferred<void>; readonly relayHookBaseUrl?: string } = {},
163 + options: {
164 + readonly onFrame?: () => void;
165 + readonly frameGate?: {
166 + readonly started: Deferred.Deferred<void>;
167 + readonly release: Deferred.Deferred<void>;
168 + };
169 + readonly completedLaunches?: Set<string>;
170 + readonly failLaunches?: { remaining: number };
171 + readonly gate?: Deferred.Deferred<void>;
172 + readonly gateForTask?: (input: LaunchInput) => Deferred.Deferred<void> | undefined;
173 + readonly relayHookBaseUrl?: string;
174 + readonly sql?: SqlClient.SqlClient;
175 + readonly workers?: Map<string, Effect.Effect<void>>;
176 + } = {},
177 ) =>
178 Effect.gen(function* () {
179 const launches = yield* Queue.unbounded<LaunchInput>();
180 const secretsByRef = new Map<string, string>();
181 const layerDependencies = Layer.mergeAll(
79 - NodePlatformCrypto.layer,
80 - Scheduler.layer,
182 + options.frameGate === undefined && options.onFrame === undefined
183 + ? NodePlatformCrypto.layer
184 + : Layer.effect(
185 + Crypto.Crypto,
186 + Effect.gen(function* () {
187 + const crypto = yield* Crypto.Crypto;
188 + return {
189 + ...crypto,
190 + randomBytes: (size: number) =>
191 + Effect.suspend(() => {
192 + if (size === 6) options.onFrame?.();
193 + return size === 6 && options.frameGate !== undefined
194 + ? Deferred.succeed(options.frameGate.started, undefined).pipe(
195 + Effect.andThen(Deferred.await(options.frameGate.release)),
196 + Effect.andThen(crypto.randomBytes(size)),
197 + )
198 + : crypto.randomBytes(size);
199 + }),
200 + };
201 + }),
202 + ).pipe(Layer.provide(NodePlatformCrypto.layer)),
203 + options.workers === undefined
204 + ? Scheduler.layer
205 + : Layer.succeed(Scheduler.Scheduler, {
206 + register: <E, R>(name: string, work: Effect.Effect<void, E, R>) =>
207 + Effect.gen(function* () {
208 + const context = yield* Effect.context<R>();
209 + options.workers!.set(name, work.pipe(Effect.provideContext(context), Effect.orDie));
210 + }),
211 + }),
212 Layer.mock(ThreadLaunchService.ThreadLaunchService)({
213 launch: (input) =>
83 - Queue.offer(launches, input).pipe(
84 - Effect.andThen(options.gate ? Deferred.await(options.gate) : Effect.void),
85 - Effect.as({ threadId: "thread-1", resumed: false } as never),
86 - ),
214 + Effect.suspend(() => {
215 + if (options.completedLaunches?.has(input.commandId))
216 + return Effect.succeed({ threadId: "thread-1", resumed: false } as never);
217 + options.completedLaunches?.add(input.commandId);
218 + return Queue.offer(launches, input).pipe(
219 + Effect.andThen(
220 + (options.gateForTask?.(input) ?? options.gate)
221 + ? Deferred.await((options.gateForTask?.(input) ?? options.gate)!)
222 + : Effect.void,
223 + ),
224 + Effect.andThen(
225 + Effect.sync(() => {
226 + if (options.failLaunches && options.failLaunches.remaining-- > 0)
227 + throw new Error("injected start failure");
228 + }),
229 + ),
230 + Effect.as({ threadId: "thread-1", resumed: false } as never),
231 + );
232 + }),
233 }),
234 Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({
235 namedProjectsRoot: "/projects",
@@ -108,8 +254,18 @@ const withService = <A, E>(
254 const service = yield* ScheduledTaskService.ScheduledTaskService;
255 const sql = yield* SqlClient.SqlClient;
256 return yield* body({ service, launches, sql, secretsByRef });
111 - }).pipe(Effect.provide(ScheduledTaskService.layer.pipe(Layer.provide(layerDependencies))));
112 - }).pipe(Effect.provide(SqlitePersistence.layerMemory));
257 + }).pipe(
258 + Effect.provide(
259 + Layer.fresh(ScheduledTaskService.layer.pipe(Layer.provide(layerDependencies))),
260 + ),
261 + );
262 + }).pipe(
263 + Effect.provide(
264 + options.sql === undefined
265 + ? SqlitePersistence.layerMemory
266 + : Layer.succeed(SqlClient.SqlClient, options.sql),
267 + ),
268 + );
269
270 it.effect("dispatches exactly the rendered prompt and logs the delivery", () =>
271 withService(({ service, launches }) =>
@@ -132,7 +288,10 @@ it.effect("dispatches exactly the rendered prompt and logs the delivery", () =>
288 `scheduled-task:${task.id}:webhook:${result._tag === "accepted" ? result.deliveryId : ""}`,
289 );
290
135 - const { deliveries } = yield* service.listWebhookDeliveries({ id: task.id });
291 + const { deliveries } = yield* service.listWebhookDeliveries({
292 + id: task.id,
293 + acceptAnyOutcome: true,
294 + });
295 assert.equal(deliveries.length, 1);
296 assert.equal(deliveries[0]?.outcome, "accepted");
297 const { delivery } = yield* service.getWebhookDelivery({
@@ -188,7 +347,11 @@ it.effect("answers not found for a wrong token or unknown hook without logging",
347 assert.equal(wrongToken._tag, "not_found");
348 const unknown = yield* service.triggerWebhook(requestFor(task, { hookId: "missing" }));
349 assert.equal(unknown._tag, "not_found");
191 - assert.equal((yield* service.listWebhookDeliveries({ id: task.id })).deliveries.length, 0);
350 + assert.equal(
351 + (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true })).deliveries
352 + .length,
353 + 0,
354 + );
355 }),
356 ),
357 );
@@ -253,9 +416,10 @@ it.effect("checks the configured signature and keeps the secret write-only", ()
416 );
417 assert.isTrue(resaved.task.webhook?.hasSecret);
418
256 - const outcomes = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries.map(
257 - (delivery) => [delivery.outcome, delivery.signatureVerified],
258 - );
419 + const outcomes = (yield* service.listWebhookDeliveries({
420 + id: task.id,
421 + acceptAnyOutcome: true,
422 + })).deliveries.map((delivery) => [delivery.outcome, delivery.signatureVerified]);
423 assert.deepEqual(outcomes.toSorted(), [
424 ["accepted", true],
425 ["rejected_signature", false],
@@ -300,23 +464,38 @@ it.effect("queues a burst of deliveries instead of dropping them", () =>
464 }),
465 );
466
303 -it.effect("rate limits a hook past 60 deliveries a minute", () =>
304 - withService(({ service }) =>
305 - Effect.gen(function* () {
306 - const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
307 - const results = yield* Effect.forEach(Array.from({ length: 61 }), () =>
308 - service.triggerWebhook(requestFor(task)),
309 - );
310 - assert.equal(results.at(-2)?._tag, "disabled");
311 - assert.equal(results.at(-1)?._tag, "rate_limited");
312 - yield* Effect.forEach([1, 2, 3], () => service.triggerWebhook(requestFor(task)));
313 - const outcomes = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries.map(
314 - (delivery) => delivery.outcome,
315 - );
316 - assert.equal(outcomes.filter((outcome) => outcome === "rate_limited").length, 1);
317 - }),
318 - ),
319 -);
467 +it.effect("rate limits a hook past 60 deliveries a minute", () => {
468 + let frames = 0;
469 + return withService(
470 + ({ service, launches }) =>
471 + Effect.gen(function* () {
472 + const { task } = yield* service.upsert(yield* webhookTaskInput());
473 + const results = yield* Effect.forEach(Array.from({ length: 61 }), () =>
474 + service
475 + .triggerWebhook(requestFor(task))
476 + .pipe(
477 + Effect.tap((result) =>
478 + result._tag === "accepted" ? Queue.take(launches) : Effect.void,
479 + ),
480 + ),
481 + );
482 + assert.equal(results.at(-2)?._tag, "accepted");
483 + assert.equal(results.at(-1)?._tag, "rate_limited");
484 + yield* Effect.forEach([1, 2, 3], () => service.triggerWebhook(requestFor(task)));
485 + const outcomes = (yield* service.listWebhookDeliveries({
486 + id: task.id,
487 + acceptAnyOutcome: true,
488 + })).deliveries.map((delivery) => delivery.outcome);
489 + assert.equal(outcomes.filter((outcome) => outcome === "rate_limited").length, 1);
490 + assert.equal(frames, 60);
491 + }),
492 + {
493 + onFrame: () => {
494 + frames++;
495 + },
496 + },
497 + );
498 +});
499
500 it.effect("a save carrying a stale token cannot undo a rotation", () =>
501 withService(({ service }) =>
@@ -337,7 +516,10 @@ it.effect("keeps the newest 50 deliveries when they share a timestamp", () =>
516 const ids = yield* Effect.forEach(Array.from({ length: 55 }), (_, index) =>
517 service.triggerWebhook(requestFor(task, { query: `n=${index}` })).pipe(Effect.as(index)),
518 );
340 - const { deliveries } = yield* service.listWebhookDeliveries({ id: task.id });
519 + const { deliveries } = yield* service.listWebhookDeliveries({
520 + id: task.id,
521 + acceptAnyOutcome: true,
522 + });
523 assert.equal(deliveries.length, 50);
524 const first = yield* service.getWebhookDelivery({
525 id: task.id,
@@ -368,7 +550,10 @@ it.effect("keeps credential headers and query values out of the delivery log", (
550 },
551 }),
552 );
371 - const [summary] = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries;
553 + const [summary] = (yield* service.listWebhookDeliveries({
554 + id: task.id,
555 + acceptAnyOutcome: true,
556 + })).deliveries;
557 const { delivery } = yield* service.getWebhookDelivery({
558 id: task.id,
559 deliveryId: summary!.id,
@@ -479,7 +664,10 @@ it.effect("caps deliveries waiting behind a stuck run", () =>
664 );
665 assert.equal(waiting.filter((result) => result._tag === "accepted").length, 19);
666 assert.equal(waiting.at(-1)?._tag, "rate_limited");
482 - const logged = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries;
667 + const logged = (yield* service.listWebhookDeliveries({
668 + id: task.id,
669 + acceptAnyOutcome: true,
670 + })).deliveries;
671 assert.equal(logged.length, 20);
672 yield* Deferred.succeed(gate, undefined);
673 }),
@@ -495,7 +683,10 @@ it.effect("logs a body's first 64 KiB by bytes, not characters", () =>
683 const text = "界".repeat(30_000);
684 const body = new TextEncoder().encode(text);
685 yield* service.triggerWebhook(requestFor(task, { body, bodyText: text }));
498 - const [summary] = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries;
686 + const [summary] = (yield* service.listWebhookDeliveries({
687 + id: task.id,
688 + acceptAnyOutcome: true,
689 + })).deliveries;
690 const { delivery } = yield* service.getWebhookDelivery({
691 id: task.id,
692 deliveryId: summary!.id,
@@ -559,7 +750,8 @@ it.effect("a held request already delivered directly runs only once", () =>
750 assert.equal(direct._tag, "accepted");
751 assert.deepEqual(replayed, { ...direct, outcome: "duplicate" } as typeof replayed);
752 yield* Queue.take(launches);
562 - const logged = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries;
753 + const logged = (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true }))
754 + .deliveries;
755 assert.equal(logged.length, 1);
756 }),
757 ),
@@ -580,7 +772,9 @@ it.effect("relay retries recover failed acceptance without releasing another req
772 const error = yield* service.triggerWebhook(request).pipe(Effect.flip);
773 assert.equal(error.message, "Could not record webhook delivery.");
774 assert.equal(yield* Queue.size(launches), 0);
583 - assert.isEmpty((yield* service.listWebhookDeliveries({ id: task.id })).deliveries);
775 + assert.isEmpty(
776 + (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true })).deliveries,
777 + );
778 yield* sql`DROP TRIGGER fail_webhook_delivery_log`;
779
780 const retried = yield* Effect.all(
@@ -606,7 +800,11 @@ it.effect("relay retries recover failed acceptance without releasing another req
800 const replayed = yield* service.triggerWebhook(request);
801 assert.equal(replayed._tag === "accepted" ? replayed.outcome : replayed._tag, "duplicate");
802 assert.equal(yield* Queue.size(launches), 0);
609 - assert.equal((yield* service.listWebhookDeliveries({ id: task.id })).deliveries.length, 1);
803 + assert.equal(
804 + (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true })).deliveries
805 + .length,
806 + 1,
807 + );
808 }),
809 ),
810 );
@@ -657,9 +855,9 @@ it.effect("a held request runs once even after the log has trimmed it", () =>
855 it.effect("a rate-limited held request can run on a later pass", () =>
856 withService(({ service, launches }) =>
857 Effect.gen(function* () {
660 - const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
858 + const { task } = yield* service.upsert(yield* webhookTaskInput());
859 yield* Effect.forEach(Array.from({ length: 60 }), () =>
662 - service.triggerWebhook(requestFor(task)),
860 + service.triggerWebhook(requestFor(task)).pipe(Effect.tap(() => Queue.take(launches))),
861 );
862 const limited = yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "later" }));
863 assert.equal(limited._tag, "rate_limited");
@@ -678,7 +876,10 @@ it.effect("logs a held request at the time the relay received it", () =>
876 const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
877 const receivedAt = DateTime.formatIso(DateTime.subtract(yield* DateTime.now, { minutes: 5 }));
878 yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "relay-2", receivedAt }));
681 - const [delivery] = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries;
879 + const [delivery] = (yield* service.listWebhookDeliveries({
880 + id: task.id,
881 + acceptAnyOutcome: true,
882 + })).deliveries;
883 assert.equal(delivery?.receivedAt, receivedAt);
884 }),
885 ),
@@ -699,7 +900,10 @@ it.effect("a receive time in the future counts as now", () =>
900 );
901 assert.equal(result._tag, "accepted");
902 yield* Queue.take(launches);
702 - const [delivery] = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries;
903 + const [delivery] = (yield* service.listWebhookDeliveries({
904 + id: task.id,
905 + acceptAnyOutcome: true,
906 + })).deliveries;
907 assert.equal(delivery?.receivedAt, DateTime.formatIso(now));
908 }),
909 ),
@@ -723,9 +927,10 @@ it.effect("skips a held request older than the task's max age", () =>
927 requestFor(task, { relayDeliveryId: "fresh", receivedAt: fresh }),
928 );
929 assert.equal(ok._tag, "accepted");
726 - const outcomes = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries.map(
727 - (delivery) => delivery.outcome,
728 - );
930 + const outcomes = (yield* service.listWebhookDeliveries({
931 + id: task.id,
932 + acceptAnyOutcome: true,
933 + })).deliveries.map((delivery) => delivery.outcome);
934 assert.includeMembers(outcomes, ["expired", "accepted"]);
935 }),
936 ),
@@ -753,7 +958,11 @@ it.effect("deleting a task removes its delivery log", () =>
958 const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
959 yield* service.triggerWebhook(requestFor(task));
960 yield* service.delete({ id: task.id });
756 - assert.equal((yield* service.listWebhookDeliveries({ id: task.id })).deliveries.length, 0);
961 + assert.equal(
962 + (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true })).deliveries
963 + .length,
964 + 0,
965 + );
966 }),
967 ),
968 );
@@ -954,105 +1163,1602 @@ it.effect("counts each handled request by what happened to it", () =>
1163 ),
1164 );
1165
957 -it.effect("checks filters after the signature and max age, and runs only matching deliveries", () =>
1166 +it.effect("dedupes replay keys per task, persists them, and expires them after seven days", () =>
1167 + withService(
1168 + ({ service, launches, sql }) =>
1169 + Effect.gen(function* () {
1170 + const schedule = { type: "webhook", dedupe: { replayKey: "{{headers.delivery-id}}" } };
1171 + const { task } = yield* service.upsert(yield* webhookTaskInput({ schedule }));
1172 + const request = requestFor(task, { headers: { "delivery-id": "A" } });
1173 + const first = yield* service.triggerWebhook(request);
1174 + assert.equal(first._tag === "accepted" && first.outcome, "accepted");
1175 + yield* Queue.take(launches);
1176 + const replay = yield* service.triggerWebhook(request);
1177 + assert.equal(replay._tag === "accepted" && replay.outcome, "duplicate");
1178 + assert.equal(yield* Queue.size(launches), 0);
1179 + const other = yield* service.upsert(
1180 + yield* webhookTaskInput({ id: "scheduled-task:other", schedule }),
1181 + );
1182 + yield* service.triggerWebhook(requestFor(other.task, { headers: request.headers }));
1183 + yield* Queue.take(launches);
1184 + yield* withService(
1185 + ({ service: restarted }) =>
1186 + Effect.gen(function* () {
1187 + const replay = yield* restarted.triggerWebhook(request);
1188 + assert.equal(replay._tag === "accepted" && replay.outcome, "duplicate");
1189 + }),
1190 + { sql, workers: new Map() },
1191 + );
1192 + yield* TestClock.adjust("7 days");
1193 + const expired = yield* service.triggerWebhook(request);
1194 + assert.equal(expired._tag === "accepted" && expired.outcome, "accepted");
1195 + yield* Queue.take(launches);
1196 + assert.include(
1197 + (yield* service.listWebhookDeliveries({
1198 + id: task.id,
1199 + acceptAnyOutcome: true,
1200 + })).deliveries.map((d) => d.outcome),
1201 + "duplicate",
1202 + );
1203 + }),
1204 + { workers: new Map() },
1205 + ),
1206 +);
1207 +
1208 +it.effect.each([1, 3])(
1209 + "coalesces %i deliveries into one run with the latest body, headers and query",
1210 + (count) => {
1211 + const workers = new Map<string, Effect.Effect<void>>();
1212 + return withService(
1213 + ({ service, launches }) =>
1214 + Effect.gen(function* () {
1215 + const { task } = yield* service.upsert(
1216 + yield* webhookTaskInput({
1217 + prompt: "{{body.value}} {{headers.version}} {{query.version}}",
1218 + schedule: {
1219 + type: "webhook",
1220 + dedupe: { coalesceKey: "{{body.key}}", coalesceWindowSeconds: 10 },
1221 + },
1222 + }),
1223 + );
1224 + for (let value = 1; value <= count; value++) {
1225 + const bodyText = JSON.stringify({ key: "B", value });
1226 + yield* service.triggerWebhook(
1227 + requestFor(task, {
1228 + bodyText,
1229 + body: new TextEncoder().encode(bodyText),
1230 + headers: { version: String(value) },
1231 + query: `version=${value}`,
1232 + }),
1233 + );
1234 + if (value < count) yield* TestClock.adjust("2 seconds");
1235 + }
1236 + const owner = (yield* service.listWebhookDeliveries({
1237 + id: task.id,
1238 + acceptAnyOutcome: true,
1239 + })).deliveries.find((delivery) => delivery.outcome === "accepted")!;
1240 + assert.isTrue(owner.pendingWindow);
1241 + const beforeDispatch = (yield* service.getWebhookDelivery({
1242 + id: task.id,
1243 + deliveryId: owner.id,
1244 + acceptAnyOutcome: true,
1245 + })).delivery;
1246 + assert.isTrue(beforeDispatch.pendingWindow);
1247 + assert.equal(beforeDispatch.body, JSON.stringify({ key: "B", value: count }));
1248 + assert.equal(beforeDispatch.headers.version, String(count));
1249 + assert.equal(beforeDispatch.query, `version=${count}`);
1250 + assert.equal(
1251 + beforeDispatch.bodyBytes,
1252 + new TextEncoder().encode(beforeDispatch.body).byteLength,
1253 + );
1254 + assert.equal(yield* Queue.size(launches), 0);
1255 + yield* workers.get("webhook-coalescing")!;
1256 + assert.equal(yield* Queue.size(launches), 0);
1257 + yield* TestClock.adjust(count === 1 ? "10 seconds" : "6 seconds");
1258 + yield* workers.get("webhook-coalescing")!;
1259 + const launched = yield* Queue.take(launches);
1260 + assert.equal(
1261 + (launched.initialMessage!.text.match(new RegExp(`>${count}<`, "g")) ?? []).length,
1262 + 3,
1263 + );
1264 + yield* workers.get("webhook-coalescing")!;
1265 + assert.equal(yield* Queue.size(launches), 0);
1266 + const outcomes = (yield* service.listWebhookDeliveries({
1267 + id: task.id,
1268 + acceptAnyOutcome: true,
1269 + })).deliveries.map((d) => d.outcome);
1270 + const afterDispatch = (yield* service.getWebhookDelivery({
1271 + id: task.id,
1272 + deliveryId: owner.id,
1273 + acceptAnyOutcome: true,
1274 + })).delivery;
1275 + assert.isFalse(afterDispatch.pendingWindow);
1276 + assert.equal(afterDispatch.body, beforeDispatch.body);
1277 + assert.include(afterDispatch.renderedPrompt!, `>${count}<`);
1278 + assert.equal(outcomes.filter((o) => o === "accepted").length, 1);
1279 + assert.equal(outcomes.filter((o) => o === "coalesced").length, count - 1);
1280 + }),
1281 + { workers },
1282 + );
1283 + },
1284 +);
1285 +
1286 +it.effect("recovers a persisted coalescing window after restarting the service", () => {
1287 + const workers = new Map<string, Effect.Effect<void>>();
1288 + return withService(
1289 + ({ service, sql }) =>
1290 + Effect.gen(function* () {
1291 + const { task } = yield* service.upsert(
1292 + yield* webhookTaskInput({
1293 + schedule: {
1294 + type: "webhook",
1295 + dedupe: { coalesceKey: "{{headers.content-type}}", coalesceWindowSeconds: 10 },
1296 + },
1297 + }),
1298 + );
1299 + yield* service.triggerWebhook(requestFor(task));
1300 + yield* TestClock.adjust("4 seconds");
1301 + const restartedWorkers = new Map<string, Effect.Effect<void>>();
1302 + yield* withService(
1303 + ({ launches }) =>
1304 + Effect.gen(function* () {
1305 + yield* restartedWorkers.get("webhook-coalescing")!;
1306 + assert.equal(yield* Queue.size(launches), 0);
1307 + yield* TestClock.adjust("6 seconds");
1308 + yield* restartedWorkers.get("webhook-coalescing")!;
1309 + yield* Queue.take(launches);
1310 + yield* restartedWorkers.get("webhook-coalescing")!;
1311 + assert.equal(yield* Queue.size(launches), 0);
1312 + assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_pending`).length, 0);
1313 + }),
1314 + { sql, workers: restartedWorkers },
1315 + );
1316 + }),
1317 + { workers },
1318 + );
1319 +});
1320 +
1321 +it.effect("empty dedupe key renders run normally and malformed templates fail at save time", () =>
1322 withService(({ service, launches }) =>
1323 Effect.gen(function* () {
1324 const { task } = yield* service.upsert(
1325 yield* webhookTaskInput({
1326 schedule: {
1327 type: "webhook",
964 - signature: {
965 - header: "x-hub-signature-256",
966 - encoding: "hex",
967 - prefix: "sha256=",
968 - secret: "s3cret",
969 - },
970 - maxDeliveryAgeMinutes: 30,
971 - filters: [
972 - { path: "headers.x-github-event", op: "equals", value: "pull_request" },
973 - { path: "body.action", op: "in", value: ["opened", "reopened"] },
974 - ],
1328 + dedupe: { replayKey: "{{headers.absent}}", coalesceKey: "{{body.absent}}" },
1329 },
1330 }),
1331 );
978 - const delivery = (
979 - event: string,
980 - action: string,
981 - options: { readonly signed?: boolean; readonly receivedAt?: string } = {},
982 - ) => {
983 - const body = new TextEncoder().encode(
984 - JSON.stringify({ action, pull_request: { url: "https://github.com/org/repo/pull/45" } }),
1332 + for (let i = 0; i < 2; i++) {
1333 + const result = yield* service.triggerWebhook(requestFor(task, { headers: { absent: "" } }));
1334 + assert.equal(result._tag === "accepted" && result.outcome, "accepted");
1335 + yield* Queue.take(launches);
1336 + }
1337 + for (const replayKey of ["literal", "{{body.bad"]) {
1338 + const invalid = yield* webhookTaskInput({
1339 + schedule: { type: "webhook", dedupe: { replayKey } },
1340 + }).pipe(Effect.result);
1341 + assert.equal(invalid._tag, "Failure");
1342 + }
1343 + for (const coalesceWindowSeconds of [9, 3601, 10.5]) {
1344 + const result = yield* webhookTaskInput({
1345 + schedule: { type: "webhook", dedupe: { coalesceWindowSeconds } },
1346 + }).pipe(Effect.result);
1347 + assert.equal(result._tag, "Failure");
1348 + }
1349 + }),
1350 + ),
1351 +);
1352 +
1353 +it.effect("applies replay before coalescing and defaults the fixed window to 120 seconds", () => {
1354 + const workers = new Map<string, Effect.Effect<void>>();
1355 + return withService(
1356 + ({ service, launches }) =>
1357 + Effect.gen(function* () {
1358 + const { task } = yield* service.upsert(
1359 + yield* webhookTaskInput({
1360 + prompt: "{{body.value}}",
1361 + schedule: {
1362 + type: "webhook",
1363 + dedupe: { replayKey: "{{headers.delivery-id}}", coalesceKey: "{{body.group}}" },
1364 + },
1365 + }),
1366 );
986 - const signature = `sha256=${NodeCrypto.createHmac("sha256", "s3cret").update(body).digest("hex")}`;
987 - return requestFor(task, {
988 - headers: {
989 - "content-type": "application/json",
990 - "x-github-event": event,
991 - ...(options.signed === false ? {} : { "x-hub-signature-256": signature }),
992 - },
993 - body,
994 - bodyText: new TextDecoder().decode(body),
995 - ...(options.receivedAt === undefined
996 - ? {}
997 - : { relayDeliveryId: "old", receivedAt: options.receivedAt }),
998 - });
999 - };
1000 - const now = yield* DateTime.now;
1001 - const old = DateTime.formatIso(DateTime.subtract(now, { minutes: 31 }));
1367 + const request = (id: string, value: string) => {
1368 + const bodyText = JSON.stringify({ group: "B", value });
1369 + return requestFor(task, {
1370 + bodyText,
1371 + body: new TextEncoder().encode(bodyText),
1372 + headers: { "delivery-id": id },
1373 + });
1374 + };
1375 + yield* service.triggerWebhook(request("B1", "first"));
1376 + yield* service.triggerWebhook(request("B2", "latest"));
1377 + const replay = yield* service.triggerWebhook(
1378 + request("B2", "replay must not replace latest"),
1379 + );
1380 + assert.equal(replay._tag === "accepted" && replay.outcome, "duplicate");
1381 + yield* TestClock.adjust("119 seconds");
1382 + yield* workers.get("webhook-coalescing")!;
1383 + assert.equal(yield* Queue.size(launches), 0);
1384 + yield* TestClock.adjust("1 second");
1385 + yield* workers.get("webhook-coalescing")!;
1386 + assert.include((yield* Queue.take(launches)).initialMessage!.text, ">latest<");
1387 + }),
1388 + { workers },
1389 + );
1390 +});
1391
1003 - // A request failing a filter is still refused first for its signature or age.
1004 - const unsigned = yield* service.triggerWebhook(delivery("push", "opened", { signed: false }));
1005 - assert.equal(unsigned._tag, "rejected_signature");
1006 - const tooOld = yield* service.triggerWebhook(delivery("push", "opened", { receivedAt: old }));
1007 - assert.equal(tooOld._tag, "expired");
1392 +it.effect("rolls back replay and pending state when recording a delivery fails", () => {
1393 + const workers = new Map<string, Effect.Effect<void>>();
1394 + return withService(
1395 + ({ service, sql, launches }) =>
1396 + Effect.gen(function* () {
1397 + const { task } = yield* service.upsert(
1398 + yield* webhookTaskInput({
1399 + schedule: {
1400 + type: "webhook",
1401 + dedupe: {
1402 + replayKey: "{{headers.content-type}}",
1403 + coalesceKey: "{{headers.content-type}}",
1404 + coalesceWindowSeconds: 10,
1405 + },
1406 + },
1407 + }),
1408 + );
1409 + yield* sql`CREATE TRIGGER fail_dedupe_log BEFORE INSERT ON scheduled_task_webhook_deliveries BEGIN SELECT RAISE(FAIL, 'injected'); END`;
1410 + yield* service.triggerWebhook(requestFor(task)).pipe(Effect.flip);
1411 + assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_pending`).length, 0);
1412 + assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_replay_keys`).length, 0);
1413 + yield* sql`DROP TRIGGER fail_dedupe_log`;
1414 + const retry = yield* service.triggerWebhook(requestFor(task));
1415 + assert.equal(retry._tag === "accepted" && retry.outcome, "accepted");
1416 + yield* TestClock.adjust("10 seconds");
1417 + yield* workers.get("webhook-coalescing")!;
1418 + yield* Queue.take(launches);
1419 + }),
1420 + { workers },
1421 + );
1422 +});
1423
1009 - const wrongEvent = yield* service.triggerWebhook(delivery("push", "opened"));
1010 - const wrongAction = yield* service.triggerWebhook(delivery("pull_request", "closed"));
1011 - assert.deepInclude(wrongEvent, { _tag: "accepted", outcome: "filtered" });
1012 - assert.deepInclude(wrongAction, { _tag: "accepted", outcome: "filtered" });
1013 - assert.equal(yield* Queue.size(launches), 0);
1014 - const [listed] = (yield* service.list()).tasks;
1015 - assert.equal(listed?.runCount, 0);
1016 - assert.equal(listed?.lastRunStatus, "never");
1424 +it.effect("starts a new fixed window at the deadline without replacing the expired payload", () => {
1425 + const workers = new Map<string, Effect.Effect<void>>();
1426 + return withService(
1427 + ({ service, launches }) =>
1428 + Effect.gen(function* () {
1429 + const { task } = yield* service.upsert(
1430 + yield* webhookTaskInput({
1431 + prompt: "{{body}}",
1432 + schedule: {
1433 + type: "webhook",
1434 + dedupe: { coalesceKey: "{{headers.content-type}}", coalesceWindowSeconds: 10 },
1435 + },
1436 + }),
1437 + );
1438 + const request = (bodyText: string) =>
1439 + requestFor(task, { bodyText, body: new TextEncoder().encode(bodyText) });
1440 + yield* service.triggerWebhook(request("first"));
1441 + yield* TestClock.adjust("10 seconds");
1442 + yield* service.triggerWebhook(request("second"));
1443 + yield* workers.get("webhook-coalescing")!;
1444 + assert.include((yield* Queue.take(launches)).initialMessage!.text, ">first<");
1445 + assert.equal(yield* Queue.size(launches), 0);
1446 + yield* TestClock.adjust("10 seconds");
1447 + yield* workers.get("webhook-coalescing")!;
1448 + assert.include((yield* Queue.take(launches)).initialMessage!.text, ">second<");
1449 + }),
1450 + { workers },
1451 + );
1452 +});
1453
1018 - const matched = yield* service.triggerWebhook(delivery("pull_request", "opened"));
1019 - assert.deepInclude(matched, { _tag: "accepted", outcome: "accepted" });
1020 - assert.include((yield* Queue.take(launches)).initialMessage?.text ?? "", "pull/45");
1454 +it.effect("pausing a pending task prevents dispatch and deletion clears its dedupe state", () => {
1455 + const workers = new Map<string, Effect.Effect<void>>();
1456 + return withService(
1457 + ({ service, launches, sql }) =>
1458 + Effect.gen(function* () {
1459 + const { task } = yield* service.upsert(
1460 + yield* webhookTaskInput({
1461 + schedule: {
1462 + type: "webhook",
1463 + dedupe: {
1464 + replayKey: "{{headers.content-type}}",
1465 + coalesceKey: "{{headers.content-type}}",
1466 + coalesceWindowSeconds: 10,
1467 + },
1468 + },
1469 + }),
1470 + );
1471 + yield* service.triggerWebhook(requestFor(task));
1472 + yield* service.setEnabled({ id: task.id, enabled: false });
1473 + yield* TestClock.adjust("10 seconds");
1474 + yield* drainPendingDispatch(workers);
1475 + assert.equal(yield* Queue.size(launches), 0);
1476 + assert.equal(
1477 + (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true }))
1478 + .deliveries[0]?.outcome,
1479 + "dispatch_failed",
1480 + );
1481 + yield* service.delete({ id: task.id });
1482 + assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_replay_keys`).length, 0);
1483 + assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_pending`).length, 0);
1484 + }),
1485 + { workers },
1486 + );
1487 +});
1488
1022 - const deliveries = (yield* service.listWebhookDeliveries({
1023 - id: task.id,
1024 - acceptAnyOutcome: true,
1025 - })).deliveries;
1026 - const filtered = deliveries
1027 - .filter((entry) => entry.outcome === "filtered")
1028 - .map((entry) => [entry.signatureVerified, entry.error]);
1029 - assert.sameDeepMembers(filtered, [
1030 - [true, "Filter 1 (headers.x-github-event) did not match."],
1031 - [true, "Filter 2 (body.action) did not match."],
1032 - ]);
1033 - }),
1034 - ),
1489 +it.effect(
1490 + "caps duplicate writes at ten per minute without evicting runs or consuming their rate slots",
1491 + () =>
1492 + withService(({ service, launches, sql }) =>
1493 + Effect.gen(function* () {
1494 + const { task } = yield* service.upsert(
1495 + yield* webhookTaskInput({
1496 + schedule: { type: "webhook", dedupe: { replayKey: "{{body}}" } },
1497 + }),
1498 + );
1499 + yield* service.triggerWebhook(requestFor(task));
1500 + yield* Queue.take(launches);
1501 + const before = yield* deliveriesCounted("duplicate", "direct");
1502 + for (let i = 0; i < 75; i++) {
1503 + const result = yield* service.triggerWebhook(requestFor(task));
1504 + assert.equal(result._tag === "accepted" && result.outcome, "duplicate");
1505 + }
1506 + assert.equal((yield* deliveriesCounted("duplicate", "direct")) - before, 75);
1507 + const rows = (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true }))
1508 + .deliveries;
1509 + assert.equal(rows.filter((row) => row.outcome === "duplicate").length, 10);
1510 + assert.equal(rows.filter((row) => row.outcome === "accepted").length, 1);
1511 + const keys = yield* sql<{
1512 + replay_key: string;
1513 + }>`SELECT replay_key FROM scheduled_task_webhook_replay_keys`;
1514 + assert.match(keys[0]!.replay_key, /^[a-f0-9]{64}$/);
1515 + for (let i = 0; i < 59; i++) {
1516 + const bodyText = JSON.stringify({ i });
1517 + const result = yield* service.triggerWebhook(
1518 + requestFor(task, { bodyText, body: new TextEncoder().encode(bodyText) }),
1519 + );
1520 + assert.equal(result._tag, "accepted");
1521 + yield* Queue.take(launches);
1522 + }
1523 + }),
1524 + ),
1525 );
1526
1037 -const pullRequestFilterInput = () =>
1038 - webhookTaskInput({
1039 - schedule: {
1040 - type: "webhook",
1041 - filters: [{ path: "headers.x-github-event", op: "equals", value: "pull_request" }],
1042 - },
1043 - });
1044 -const githubEvent = (event: string) => ({
1045 - headers: { "content-type": "application/json", "x-github-event": event },
1527 +it.effect("caps coalesced history while persisting the latest overflow payload", () => {
1528 + const workers = new Map<string, Effect.Effect<void>>();
1529 + return withService(
1530 + ({ service, sql, launches }) =>
1531 + Effect.gen(function* () {
1532 + const { task } = yield* service.upsert(
1533 + yield* webhookTaskInput({
1534 + prompt: "{{body.value}}",
1535 + schedule: {
1536 + type: "webhook",
1537 + dedupe: { replayKey: "{{headers.delivery-id}}", coalesceKey: "{{body.group}}" },
1538 + },
1539 + }),
1540 + );
1541 + const request = (i: number) => {
1542 + const bodyText = JSON.stringify({ group: "B", value: i });
1543 + return requestFor(task, {
1544 + bodyText,
1545 + body: new TextEncoder().encode(bodyText),
1546 + headers: { "delivery-id": String(i) },
1547 + });
1548 + };
1549 + for (let i = 0; i < 40; i++)
1550 + assert.equal((yield* service.triggerWebhook(request(i)))._tag, "accepted");
1551 + assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_replay_keys`).length, 40);
1552 + const rows = (yield* service.listWebhookDeliveries({
1553 + id: task.id,
1554 + acceptAnyOutcome: true,
1555 + })).deliveries;
1556 + assert.equal(rows.filter((row) => row.outcome === "coalesced").length, 10);
1557 + assert.equal(rows.filter((row) => row.outcome === "accepted").length, 1);
1558 + const pending = yield* sql<{
1559 + coalesce_key: string;
1560 + }>`SELECT coalesce_key FROM scheduled_task_webhook_pending`;
1561 + assert.match(pending[0]!.coalesce_key, /^[a-f0-9]{64}$/);
1562 + yield* TestClock.adjust("120 seconds");
1563 + yield* drainPendingDispatch(workers);
1564 + assert.include((yield* Queue.take(launches)).initialMessage!.text, ">39<");
1565 + assert.equal(yield* Queue.size(launches), 0);
1566 + }),
1567 + { workers },
1568 + );
1569 });
1570
1048 -it.effect("filtered deliveries take no run slot and log at most 10 rows a minute", () =>
1571 +it.effect("a partially missing key is empty even with a literal prefix", () =>
1572 withService(({ service, launches }) =>
1573 Effect.gen(function* () {
1051 - const { task } = yield* service.upsert(yield* pullRequestFilterInput());
1052 - const ignored = yield* Effect.forEach(Array.from({ length: 70 }), () =>
1053 - service.triggerWebhook(requestFor(task, githubEvent("push"))),
1574 + const { task } = yield* service.upsert(
1575 + yield* webhookTaskInput({
1576 + schedule: {
1577 + type: "webhook",
1578 + dedupe: { replayKey: "gh:{{headers.absent}}", coalesceKey: "pr:{{body.absent}}" },
1579 + },
1580 + }),
1581 );
1055 - assert.isTrue(
1582 + for (let i = 0; i < 2; i++) {
1583 + const result = yield* service.triggerWebhook(requestFor(task, { headers: { absent: "" } }));
1584 + assert.equal(result._tag === "accepted" && result.outcome, "accepted");
1585 + yield* Queue.take(launches);
1586 + }
1587 + }),
1588 + ),
1589 +);
1590 +
1591 +it.effect("queue-full deliveries release replay keys so a later retry runs", () =>
1592 + Effect.gen(function* () {
1593 + const gate = yield* Deferred.make<void>();
1594 + yield* withService(
1595 + ({ service, launches }) =>
1596 + Effect.gen(function* () {
1597 + const { task } = yield* service.upsert(
1598 + yield* webhookTaskInput({
1599 + schedule: { type: "webhook", dedupe: { replayKey: "{{headers.delivery-id}}" } },
1600 + }),
1601 + );
1602 + const request = (i: number) =>
1603 + requestFor(task, { headers: { "delivery-id": String(i) } });
1604 + for (let i = 0; i < 20; i++) yield* service.triggerWebhook(request(i));
1605 + yield* Queue.take(launches);
1606 + const full = yield* service.triggerWebhook(request(20));
1607 + assert.equal(full._tag === "rate_limited" && full.outcome, "queue_full");
1608 + yield* Deferred.succeed(gate, undefined);
1609 + for (let i = 1; i < 20; i++) yield* Queue.take(launches);
1610 + const retry = yield* service.triggerWebhook(request(20));
1611 + assert.equal(retry._tag === "accepted" && retry.outcome, "accepted");
1612 + yield* Queue.take(launches);
1613 + }),
1614 + { gate },
1615 + );
1616 + }),
1617 +);
1618 +
1619 +it.effect.each(["task_json", "request_json"] as const)(
1620 + "removes corrupt pending %s instead of retrying forever",
1621 + (column) => {
1622 + const workers = new Map<string, Effect.Effect<void>>();
1623 + return withService(
1624 + ({ service, sql }) =>
1625 + Effect.gen(function* () {
1626 + const { task } = yield* service.upsert(
1627 + yield* webhookTaskInput({
1628 + schedule: {
1629 + type: "webhook",
1630 + dedupe: {
1631 + replayKey: "{{body}}",
1632 + coalesceKey: "{{body}}",
1633 + coalesceWindowSeconds: 10,
1634 + },
1635 + },
1636 + }),
1637 + );
1638 + yield* service.triggerWebhook(requestFor(task));
1639 + yield* sql`UPDATE scheduled_task_webhook_pending SET ${sql(column)} = 'broken'`;
1640 + yield* TestClock.adjust("10 seconds");
1641 + yield* drainPendingDispatch(workers);
1642 + assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_pending`).length, 0);
1643 + assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_replay_keys`).length, 0);
1644 + assert.equal(
1645 + (yield* service.listWebhookDeliveries({ id: task.id })).deliveries[0]!.outcome,
1646 + "dispatch_failed",
1647 + );
1648 + yield* workers.get("webhook-coalescing")!;
1649 + }),
1650 + { workers },
1651 + );
1652 + },
1653 +);
1654 +
1655 +it.effect(
1656 + "restarting after a coalesced launch before pending deletion does not launch or count it twice",
1657 + () => {
1658 + const workers = new Map<string, Effect.Effect<void>>();
1659 + return withService(
1660 + ({ service, sql, launches }) =>
1661 + Effect.gen(function* () {
1662 + const { task } = yield* service.upsert(
1663 + yield* webhookTaskInput({
1664 + schedule: {
1665 + type: "webhook",
1666 + dedupe: { coalesceKey: "{{body}}", coalesceWindowSeconds: 10 },
1667 + },
1668 + }),
1669 + );
1670 + const accepted = yield* service.triggerWebhook(requestFor(task));
1671 + yield* sql`CREATE TRIGGER fail_pending_delete BEFORE DELETE ON scheduled_task_webhook_pending BEGIN SELECT RAISE(FAIL,'crash after launch'); END`;
1672 + yield* TestClock.adjust("10 seconds");
1673 + yield* drainPendingDispatch(workers);
1674 + const launch = yield* Queue.take(launches);
1675 + assert.equal(
1676 + launch.commandId,
1677 + `scheduled-task:${task.id}:webhook:${accepted._tag === "accepted" && accepted.deliveryId}`,
1678 + );
1679 + assert.equal((yield* service.list()).tasks[0]!.runCount, 1);
1680 + assert.equal(
1681 + (yield* sql`SELECT * FROM scheduled_task_webhook_pending WHERE state = 'completed'`)
1682 + .length,
1683 + 1,
1684 + );
1685 + yield* sql`DROP TRIGGER fail_pending_delete`;
1686 + const restartedWorkers = new Map<string, Effect.Effect<void>>();
1687 + yield* withService(
1688 + ({ service: restarted, launches: secondLaunches }) =>
1689 + Effect.gen(function* () {
1690 + yield* drainPendingDispatch(restartedWorkers);
1691 + assert.equal(yield* Queue.size(secondLaunches), 0);
1692 + assert.equal((yield* restarted.list()).tasks[0]!.runCount, 1);
1693 + assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_pending`).length, 0);
1694 + }),
1695 + { sql, workers: restartedWorkers },
1696 + );
1697 + }),
1698 + { workers },
1699 + );
1700 + },
1701 +);
1702 +
1703 +it.effect("many windows on a busy task share one worker without delaying another task", () =>
1704 + Effect.gen(function* () {
1705 + const gate = yield* Deferred.make<void>();
1706 + const workers = new Map<string, Effect.Effect<void>>();
1707 + yield* withService(
1708 + ({ service, launches, sql }) =>
1709 + Effect.gen(function* () {
1710 + const schedule = {
1711 + type: "webhook",
1712 + dedupe: { coalesceKey: "{{body.group}}", coalesceWindowSeconds: 10 },
1713 + };
1714 + const { task: a } = yield* service.upsert(
1715 + yield* webhookTaskInput({ id: "scheduled-task:a", title: "A", schedule }),
1716 + );
1717 + const { task: b } = yield* service.upsert(
1718 + yield* webhookTaskInput({ id: "scheduled-task:b", title: "B", schedule }),
1719 + );
1720 + yield* service.triggerWebhook(requestFor(a));
1721 + assert.equal((yield* Queue.take(launches)).title, "A");
1722 + const pendingRequest = (task: typeof a, group: string) => {
1723 + const bodyText = JSON.stringify({ group });
1724 + return requestFor(task, { bodyText, body: new TextEncoder().encode(bodyText) });
1725 + };
1726 + for (let i = 0; i < 19; i++) {
1727 + const result = yield* service.triggerWebhook(pendingRequest(a, String(i)));
1728 + assert.equal(result._tag === "accepted" && result.outcome, "accepted");
1729 + }
1730 + const overflow = pendingRequest(a, "overflow");
1731 + const full = yield* service.triggerWebhook({ ...overflow, relayDeliveryId: "overflow" });
1732 + assert.equal(full._tag === "rate_limited" && full.outcome, "queue_full");
1733 + assert.equal((yield* service.triggerWebhook(requestFor(a)))._tag, "rate_limited");
1734 + yield* service.triggerWebhook(pendingRequest(b, "B"));
1735 + let dispatchWorkers = 0;
1736 + const drained = yield* Deferred.make<void>();
1737 + let finishedWorkers = 0;
1738 + const tracer = Tracer.make({
1739 + span: (options) => {
1740 + if (options.name === "ScheduledTaskService.dispatchPendingWebhook") dispatchWorkers++;
1741 + const span = new Tracer.NativeSpan(options);
1742 + const end = span.end.bind(span);
1743 + span.end = (...args) => {
1744 + end(...args);
1745 + if (
1746 + options.name === "ScheduledTaskService.dispatchPendingWebhook" &&
1747 + ++finishedWorkers === 2
1748 + )
1749 + Deferred.doneUnsafe(drained, Effect.void);
1750 + };
1751 + return span;
1752 + },
1753 + });
1754 + yield* TestClock.adjust("10 seconds");
1755 + yield* workers.get("webhook-coalescing")!.pipe(Effect.withTracer(tracer));
1756 + assert.equal((yield* Queue.take(launches)).title, "B");
1757 + for (let i = 0; i < 10; i++)
1758 + yield* workers.get("webhook-coalescing")!.pipe(Effect.withTracer(tracer));
1759 + assert.equal(dispatchWorkers, 2);
1760 + assert.equal(
1761 + (yield* sql`SELECT 1 FROM scheduled_task_webhook_pending WHERE task_id = ${a.id}`)
1762 + .length,
1763 + 19,
1764 + );
1765 + yield* Deferred.succeed(gate, undefined);
1766 + yield* Deferred.await(drained);
1767 + for (let i = 0; i < 19; i++) assert.equal((yield* Queue.take(launches)).title, "A");
1768 + const retried = yield* service.triggerWebhook({
1769 + ...overflow,
1770 + relayDeliveryId: "overflow",
1771 + });
1772 + assert.equal(retried._tag === "accepted" && retried.outcome, "accepted");
1773 + yield* TestClock.adjust("10 seconds");
1774 + yield* drainPendingDispatch(workers);
1775 + assert.equal((yield* Queue.take(launches)).title, "A");
1776 + }),
1777 + { workers, gateForTask: (input) => (input.title === "A" ? gate : undefined) },
1778 + );
1779 + }),
1780 +);
1781 +
1782 +it.effect("guards bad-signature floods separately from valid run admission", () =>
1783 + withService(({ service, sql }) =>
1784 + Effect.gen(function* () {
1785 + const { task } = yield* service.upsert(
1786 + yield* webhookTaskInput({
1787 + schedule: {
1788 + type: "webhook",
1789 + signature: { header: "signature", encoding: "hex", prefix: "", secret: "secret" },
1790 + },
1791 + }),
1792 + );
1793 + const results = yield* Effect.forEach(Array.from({ length: 75 }), () =>
1794 + service.triggerWebhook(requestFor(task)),
1795 + );
1796 + assert.equal(results.filter((result) => result._tag === "rejected_signature").length, 60);
1797 + assert.equal(results.filter((result) => result._tag === "rate_limited").length, 15);
1798 + assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_deliveries`).length, 51);
1799 + }),
1800 + ),
1801 +);
1802 +
1803 +it.effect("start failures release both replay keys and relay claims for redelivery", () =>
1804 + withService(
1805 + ({ service, sql, launches }) =>
1806 + Effect.gen(function* () {
1807 + const { task } = yield* service.upsert(
1808 + yield* webhookTaskInput({
1809 + schedule: { type: "webhook", dedupe: { replayKey: "{{body}}" } },
1810 + }),
1811 + );
1812 + const request = requestFor(task, { relayDeliveryId: "failed-start" });
1813 + yield* waitForSpan(
1814 + service.triggerWebhook(request),
1815 + "ScheduledTaskService.runWebhookDelivery",
1816 + );
1817 + yield* Queue.take(launches);
1818 + assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_replay_keys`).length, 0);
1819 + assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_relay_deliveries`).length, 0);
1820 + const retry = yield* waitForSpan(
1821 + service.triggerWebhook(request),
1822 + "ScheduledTaskService.runWebhookDelivery",
1823 + );
1824 + assert.equal(retry._tag === "accepted" && retry.outcome, "accepted");
1825 + yield* Queue.take(launches);
1826 + assert.equal((yield* service.list()).tasks[0]!.lastRunStatus, "succeeded");
1827 + }),
1828 + { failLaunches: { remaining: 1 } },
1829 + ),
1830 +);
1831 +
1832 +it.effect("bounds oversized prompt floods without evicting run or filtered history", () =>
1833 + withService(({ service, sql, launches }) =>
1834 + Effect.gen(function* () {
1835 + const { task } = yield* service.upsert(yield* webhookTaskInput({ prompt: "{{body}}" }));
1836 + const accepted = yield* service.triggerWebhook(requestFor(task));
1837 + yield* Queue.take(launches);
1838 + assert.equal(accepted._tag, "accepted");
1839 + // A sibling filter step owns these rows; suppression must preserve its bucket too.
1840 + yield* sql`INSERT INTO scheduled_task_webhook_deliveries
1841 + SELECT 'filtered-fixture', task_id, received_at, method, query, headers_json, body,
1842 + body_bytes, body_truncated, 'filtered', signature_verified, missing_fields_json, rendered_prompt, error
1843 + FROM scheduled_task_webhook_deliveries LIMIT 1`;
1844 + const bodyText = "x".repeat(200_000);
1845 + const request = requestFor(task, { bodyText, body: new TextEncoder().encode(bodyText) });
1846 + const outcomes = yield* Effect.forEach(Array.from({ length: 75 }), () =>
1847 + service.triggerWebhook(request),
1848 + );
1849 + assert.equal(outcomes.filter((result) => result._tag === "accepted").length, 60);
1850 + assert.equal(outcomes.filter((result) => result._tag === "rate_limited").length, 15);
1851 + const rows = yield* sql<{
1852 + outcome: string;
1853 + }>`SELECT outcome FROM scheduled_task_webhook_deliveries`;
1854 + assert.equal(rows.filter((row) => row.outcome === "accepted").length, 1);
1855 + assert.equal(rows.filter((row) => row.outcome === "filtered").length, 1);
1856 + assert.equal(rows.length, 22);
1857 + assert.equal(yield* Queue.size(launches), 0);
1858 + yield* TestClock.adjust("61 seconds");
1859 + assert.equal((yield* service.triggerWebhook(requestFor(task)))._tag, "accepted");
1860 + yield* Queue.take(launches);
1861 + }),
1862 + ),
1863 +);
1864 +
1865 +it.effect("single delivery lookup hides new outcomes unless the client opts in", () =>
1866 + withService(({ service, launches }) =>
1867 + Effect.gen(function* () {
1868 + const { task } = yield* service.upsert(
1869 + yield* webhookTaskInput({
1870 + schedule: {
1871 + type: "webhook",
1872 + dedupe: { replayKey: "{{body}}", coalesceKey: "{{body.group}}" },
1873 + },
1874 + }),
1875 + );
1876 + yield* service.triggerWebhook(requestFor(task));
1877 + yield* Queue.take(launches);
1878 + yield* service.triggerWebhook(requestFor(task));
1879 + for (const value of [1, 2]) {
1880 + const bodyText = JSON.stringify({ group: "B", value });
1881 + yield* service.triggerWebhook(
1882 + requestFor(task, { bodyText, body: new TextEncoder().encode(bodyText) }),
1883 + );
1884 + }
1885 + const rows = (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true }))
1886 + .deliveries;
1887 + for (const row of rows.filter(
1888 + (row) => row.outcome === "duplicate" || row.outcome === "coalesced",
1889 + )) {
1890 + const input = { id: task.id, deliveryId: row.id };
1891 + const hidden = yield* service.getWebhookDelivery(input).pipe(Effect.result);
1892 + assert.equal(hidden._tag, "Failure");
1893 + assert.equal(
1894 + (yield* service.getWebhookDelivery({ ...input, acceptAnyOutcome: true })).delivery
1895 + .outcome,
1896 + row.outcome,
1897 + );
1898 + }
1899 + assert.equal(
1900 + rows.filter((row) => row.outcome === "duplicate" || row.outcome === "coalesced").length,
1901 + 2,
1902 + );
1903 + }),
1904 + ),
1905 +);
1906 +
1907 +it.effect("main retention keeps fifty runs and preserves filtered and dedupe buckets", () =>
1908 + withService(({ service, sql, launches }) =>
1909 + Effect.gen(function* () {
1910 + const { task } = yield* service.upsert(yield* webhookTaskInput());
1911 + yield* service.triggerWebhook(requestFor(task));
1912 + yield* Queue.take(launches);
1913 + for (const outcome of ["filtered", "duplicate"]) {
1914 + yield* sql`WITH RECURSIVE n(value) AS (SELECT 1 UNION ALL SELECT value + 1 FROM n WHERE value < 20)
1915 + INSERT INTO scheduled_task_webhook_deliveries
1916 + SELECT ${outcome} || n.value, d.task_id, d.received_at, d.method, d.query, d.headers_json, d.body,
1917 + d.body_bytes, d.body_truncated, ${outcome}, d.signature_verified, d.missing_fields_json, d.rendered_prompt, d.error
1918 + FROM n CROSS JOIN (SELECT * FROM scheduled_task_webhook_deliveries WHERE outcome = 'accepted' LIMIT 1) d`;
1919 + }
1920 + for (let i = 0; i < 59; i++) {
1921 + assert.equal((yield* service.triggerWebhook(requestFor(task)))._tag, "accepted");
1922 + yield* Queue.take(launches);
1923 + }
1924 + const counts = yield* sql<{
1925 + outcome: string;
1926 + count: number;
1927 + }>`SELECT outcome, COUNT(*) AS count FROM scheduled_task_webhook_deliveries GROUP BY outcome ORDER BY outcome`;
1928 + assert.deepEqual(counts, [
1929 + { outcome: "accepted", count: 50 },
1930 + { outcome: "duplicate", count: 20 },
1931 + { outcome: "filtered", count: 20 },
1932 + ]);
1933 + }),
1934 + ),
1935 +);
1936 +
1937 +it.effect(
1938 + "a valid request refunds its own reservation without expiring an interleaved rejection",
1939 + () =>
1940 + Effect.gen(function* () {
1941 + const started = yield* Deferred.make<void>();
1942 + const release = yield* Deferred.make<void>();
1943 + yield* withService(
1944 + ({ service, launches }) =>
1945 + Effect.gen(function* () {
1946 + const { task } = yield* service.upsert(
1947 + yield* webhookTaskInput({
1948 + schedule: {
1949 + type: "webhook",
1950 + signature: { header: "signature", encoding: "hex", prefix: "", secret: "secret" },
1951 + },
1952 + }),
1953 + );
1954 + const signature = NodeCrypto.createHmac("sha256", "secret")
1955 + .update(pullRequestBody)
1956 + .digest("hex");
1957 + const valid = yield* service
1958 + .triggerWebhook(requestFor(task, { headers: { signature } }))
1959 + .pipe(Effect.forkChild);
1960 + yield* Deferred.await(started);
1961 + yield* TestClock.adjust("30 seconds");
1962 + assert.equal(
1963 + (yield* service.triggerWebhook(requestFor(task)))._tag,
1964 + "rejected_signature",
1965 + );
1966 + yield* Deferred.succeed(release, undefined);
1967 + assert.equal((yield* Fiber.join(valid))._tag, "accepted");
1968 + yield* Queue.take(launches);
1969 + // Only the valid reservation would have expired now; the rejection must still count.
1970 + yield* TestClock.adjust("31 seconds");
1971 + for (let i = 0; i < 59; i++)
1972 + assert.equal(
1973 + (yield* service.triggerWebhook(requestFor(task)))._tag,
1974 + "rejected_signature",
1975 + );
1976 + assert.equal((yield* service.triggerWebhook(requestFor(task)))._tag, "rate_limited");
1977 + }),
1978 + { frameGate: { started, release } },
1979 + );
1980 + }),
1981 +);
1982 +
1983 +it.effect("keeps replay keys when dispatch succeeds but completion bookkeeping fails", () =>
1984 + withService(({ service, sql, launches }) =>
1985 + Effect.gen(function* () {
1986 + const { task } = yield* service.upsert(
1987 + yield* webhookTaskInput({
1988 + schedule: {
1989 + type: "webhook",
1990 + dedupe: { replayKey: "{{body}}" },
1991 + },
1992 + }),
1993 + );
1994 + yield* sql`CREATE TRIGGER fail_completion BEFORE UPDATE ON scheduled_tasks
1995 + WHEN NEW.last_run_status = 'succeeded' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`;
1996 + yield* waitForSpan(
1997 + service.triggerWebhook(requestFor(task)),
1998 + "ScheduledTaskService.runWebhookDelivery",
1999 + );
2000 + yield* Queue.take(launches);
2001 + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`).length, 1);
2002 + yield* sql`DROP TRIGGER fail_completion`;
2003 + const retry = yield* service.triggerWebhook(requestFor(task));
2004 + assert.equal(retry._tag === "accepted" && retry.outcome, "duplicate");
2005 + assert.equal(yield* Queue.size(launches), 0);
2006 + assert.equal((yield* service.list()).tasks[0]!.runCount, 1);
2007 + }),
2008 + ),
2009 +);
2010 +
2011 +it.effect("restart before a coalesced completion commits counts the delivery once", () =>
2012 + Effect.gen(function* () {
2013 + const gate = yield* Deferred.make<void>();
2014 + const completedLaunches = new Set<string>();
2015 + const workers = new Map<string, Effect.Effect<void>>();
2016 + yield* withService(
2017 + ({ service, sql, launches }) =>
2018 + Effect.gen(function* () {
2019 + const { task } = yield* service.upsert(
2020 + yield* webhookTaskInput({
2021 + schedule: {
2022 + type: "webhook",
2023 + dedupe: {
2024 + replayKey: "{{body}}",
2025 + coalesceKey: "{{body}}",
2026 + coalesceWindowSeconds: 10,
2027 + },
2028 + },
2029 + }),
2030 + );
2031 + yield* service.triggerWebhook(requestFor(task));
2032 + yield* TestClock.adjust("10 seconds");
2033 + yield* workers.get("webhook-coalescing")!;
2034 + const launch = yield* Queue.take(launches);
2035 + assert.isTrue(completedLaunches.has(launch.commandId));
2036 + assert.equal(
2037 + (yield* sql<{ state: string }>`SELECT state FROM scheduled_task_webhook_pending`)[0]!
2038 + .state,
2039 + "running",
2040 + );
2041 + assert.equal(
2042 + (yield* sql`SELECT 1 FROM scheduled_task_webhook_pending WHERE state = 'completed'`)
2043 + .length,
2044 + 0,
2045 + );
2046 + assert.equal((yield* service.list()).tasks[0]!.lastRunStatus, "running");
2047 + const restartedWorkers = new Map<string, Effect.Effect<void>>();
2048 + yield* withService(
2049 + ({ service: restarted, launches: recoveredLaunches }) =>
2050 + Effect.gen(function* () {
2051 + assert.equal((yield* restarted.list()).tasks[0]!.runCount, 0);
2052 + yield* drainPendingDispatch(restartedWorkers);
2053 + assert.equal(yield* Queue.size(recoveredLaunches), 0);
2054 + assert.equal((yield* restarted.list()).tasks[0]!.runCount, 1);
2055 + assert.equal((yield* restarted.list()).tasks[0]!.lastRunStatus, "succeeded");
2056 + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0);
2057 + const retry = yield* restarted.triggerWebhook(requestFor(task));
2058 + assert.equal(retry._tag === "accepted" && retry.outcome, "duplicate");
2059 + }),
2060 + { sql, workers: restartedWorkers, completedLaunches },
2061 + );
2062 + }),
2063 + { workers, gate, completedLaunches },
2064 + );
2065 + }),
2066 +);
2067 +
2068 +it.effect.each([119_000, 120_000])(
2069 + "bounds coalesced prompt logs and checks trimmed length for %i body characters",
2070 + (length) => {
2071 + const workers = new Map<string, Effect.Effect<void>>();
2072 + return withService(
2073 + ({ service, launches }) =>
2074 + Effect.gen(function* () {
2075 + const { task } = yield* service.upsert(
2076 + yield* webhookTaskInput({
2077 + prompt: `{{body.absent}}${" ".repeat(10_000)}{{body.text}}`,
2078 + schedule: {
2079 + type: "webhook",
2080 + dedupe: { coalesceKey: "{{body.group}}", coalesceWindowSeconds: 10 },
2081 + },
2082 + }),
2083 + );
2084 + const bodyText = JSON.stringify({ group: "B", text: "x".repeat(length) });
2085 + const accepted = yield* service.triggerWebhook(
2086 + requestFor(task, { bodyText, body: new TextEncoder().encode(bodyText) }),
2087 + );
2088 + assert.equal(accepted._tag, "accepted");
2089 + yield* TestClock.adjust("10 seconds");
2090 + yield* drainPendingDispatch(workers);
2091 + const delivery = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries[0]!;
2092 + if (length === 119_000) {
2093 + const launch = yield* Queue.take(launches);
2094 + assert.isAbove(launch.initialMessage!.text.length, 120_000);
2095 + assert.equal(delivery.outcome, "accepted");
2096 + const detail = yield* service.getWebhookDelivery({
2097 + id: task.id,
2098 + deliveryId: delivery.id,
2099 + });
2100 + assert.equal(detail.delivery.renderedPrompt!.length, 64 * 1024);
2101 + } else {
2102 + assert.equal(yield* Queue.size(launches), 0);
2103 + assert.equal(delivery.outcome, "dispatch_failed");
2104 + assert.equal(delivery.error, "The filled-in prompt is too long.");
2105 + }
2106 + }),
2107 + { workers },
2108 + );
2109 + },
2110 +);
2111 +
2112 +it.effect("retries coalesced completion bookkeeping without launching or counting twice", () => {
2113 + const workers = new Map<string, Effect.Effect<void>>();
2114 + return withService(
2115 + ({ service, sql, launches }) =>
2116 + Effect.gen(function* () {
2117 + const { task } = yield* service.upsert(
2118 + yield* webhookTaskInput({
2119 + schedule: {
2120 + type: "webhook",
2121 + dedupe: { replayKey: "{{body}}", coalesceKey: "{{body}}", coalesceWindowSeconds: 10 },
2122 + },
2123 + }),
2124 + );
2125 + yield* service.triggerWebhook(requestFor(task));
2126 + yield* sql`CREATE TRIGGER fail_coalesced_completion BEFORE UPDATE ON scheduled_tasks
2127 + WHEN NEW.last_run_status = 'succeeded' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`;
2128 + yield* TestClock.adjust("10 seconds");
2129 + yield* drainPendingDispatch(workers);
2130 + yield* Queue.take(launches);
2131 + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 1);
2132 + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`).length, 1);
2133 + assert.equal(
2134 + (yield* sql`SELECT 1 FROM scheduled_task_webhook_pending WHERE state = 'completed'`)
2135 + .length,
2136 + 0,
2137 + );
2138 + assert.equal((yield* service.list()).tasks[0]!.runCount, 0);
2139 + yield* sql`DROP TRIGGER fail_coalesced_completion`;
2140 + yield* drainPendingDispatch(workers);
2141 + assert.equal(yield* Queue.size(launches), 0);
2142 + assert.equal((yield* service.list()).tasks[0]!.runCount, 1);
2143 + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0);
2144 + const delivery = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries[0]!;
2145 + assert.equal(delivery.outcome, "accepted");
2146 + assert.equal(delivery.error, null);
2147 + }),
2148 + { workers, completedLaunches: new Set() },
2149 + );
2150 +});
2151 +
2152 +it.effect("propagates coalescing log failures with their task id and rolls back acceptance", () => {
2153 + const workers = new Map<string, Effect.Effect<void>>();
2154 + return withService(
2155 + ({ service, sql, launches }) =>
2156 + Effect.gen(function* () {
2157 + const { task } = yield* service.upsert(
2158 + yield* webhookTaskInput({
2159 + schedule: {
2160 + type: "webhook",
2161 + dedupe: { replayKey: "{{body}}", coalesceKey: "{{body}}", coalesceWindowSeconds: 10 },
2162 + },
2163 + }),
2164 + );
2165 + const request = requestFor(task, { relayDeliveryId: "coalescing-log-failure" });
2166 + yield* sql`CREATE TRIGGER fail_coalescing_log BEFORE INSERT ON scheduled_task_webhook_deliveries
2167 + BEGIN SELECT RAISE(FAIL, 'injected coalescing log failure'); END`;
2168 + const error = yield* service.triggerWebhook(request).pipe(Effect.flip);
2169 + assert.equal(error.message, "Could not record webhook delivery.");
2170 + assert.equal(error.taskId, task.id);
2171 + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0);
2172 + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`).length, 0);
2173 + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_relay_deliveries`).length, 0);
2174 + yield* sql`DROP TRIGGER fail_coalescing_log`;
2175 + assert.equal((yield* service.triggerWebhook(request))._tag, "accepted");
2176 + yield* TestClock.adjust("10 seconds");
2177 + yield* drainPendingDispatch(workers);
2178 + yield* Queue.take(launches);
2179 + }),
2180 + { workers },
2181 + );
2182 +});
2183 +
2184 +it.effect.each([1, 11])(
2185 + "keeps relay claims for %i coalesced updates after their window closes",
2186 + (count) => {
2187 + const workers = new Map<string, Effect.Effect<void>>();
2188 + return withService(
2189 + ({ service, sql, launches }) =>
2190 + Effect.gen(function* () {
2191 + const { task } = yield* service.upsert(
2192 + yield* webhookTaskInput({
2193 + prompt: "{{body.value}}",
2194 + schedule: {
2195 + type: "webhook",
2196 + dedupe: { coalesceKey: "{{body.group}}", coalesceWindowSeconds: 10 },
2197 + },
2198 + }),
2199 + );
2200 + const request = (value: number) => {
2201 + const bodyText = JSON.stringify({ group: "B", value });
2202 + return requestFor(task, {
2203 + relayDeliveryId: `coalesced-relay-${value}`,
2204 + bodyText,
2205 + body: new TextEncoder().encode(bodyText),
2206 + });
2207 + };
2208 + yield* service.triggerWebhook(request(0));
2209 + for (let i = 1; i <= count; i++) {
2210 + const result = yield* service.triggerWebhook(request(i));
2211 + assert.equal(result._tag === "accepted" && result.outcome, "coalesced");
2212 + }
2213 + assert.equal(
2214 + (yield* sql`SELECT 1 FROM scheduled_task_webhook_relay_deliveries`).length,
2215 + count + 1,
2216 + );
2217 + yield* TestClock.adjust("10 seconds");
2218 + yield* drainPendingDispatch(workers);
2219 + assert.include((yield* Queue.take(launches)).initialMessage!.text, `>${count}<`);
2220 + const retry = yield* service.triggerWebhook(request(count));
2221 + assert.equal(retry._tag === "accepted" && retry.outcome, "duplicate");
2222 + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0);
2223 + assert.equal((yield* service.list()).tasks[0]!.runCount, 1);
2224 + assert.equal(yield* Queue.size(launches), 0);
2225 + }),
2226 + { workers },
2227 + );
2228 + },
2229 +);
2230 +
2231 +it.effect("failed windows release every member's replay key and relay claim", () => {
2232 + const workers = new Map<string, Effect.Effect<void>>();
2233 + return withService(
2234 + ({ service, sql, launches }) =>
2235 + Effect.gen(function* () {
2236 + const { task } = yield* service.upsert(
2237 + yield* webhookTaskInput({
2238 + prompt: "{{body.value}}",
2239 + schedule: {
2240 + type: "webhook",
2241 + dedupe: {
2242 + replayKey: "{{headers.delivery-id}}",
2243 + coalesceKey: "{{body.group}}",
2244 + coalesceWindowSeconds: 10,
2245 + },
2246 + },
2247 + }),
2248 + );
2249 + const request = (id: string) => {
2250 + const bodyText = JSON.stringify({ group: "same", value: id });
2251 + return requestFor(task, {
2252 + bodyText,
2253 + body: new TextEncoder().encode(bodyText),
2254 + headers: { "delivery-id": id },
2255 + relayDeliveryId: id,
2256 + });
2257 + };
2258 + yield* service.triggerWebhook(request("A"));
2259 + for (let i = 0; i < 12; i++) yield* service.triggerWebhook(request(`B${i}`));
2260 + assert.equal(
2261 + (yield* sql`SELECT 1 FROM scheduled_task_webhook_relay_deliveries`).length,
2262 + 13,
2263 + );
2264 + yield* TestClock.adjust("10 seconds");
2265 + yield* drainPendingDispatch(workers);
2266 + yield* Queue.take(launches);
2267 + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`).length, 0);
2268 + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_relay_deliveries`).length, 0);
2269 + const retry = yield* service.triggerWebhook(request("B11"));
2270 + assert.equal(retry._tag === "accepted" && retry.outcome, "accepted");
2271 + yield* TestClock.adjust("10 seconds");
2272 + yield* drainPendingDispatch(workers);
2273 + assert.include((yield* Queue.take(launches)).initialMessage!.text, "B11");
2274 + assert.equal((yield* service.list()).tasks[0]!.runCount, 2);
2275 + }),
2276 + { workers, failLaunches: { remaining: 1 } },
2277 + );
2278 +});
2279 +
2280 +it.effect(
2281 + "rejects a twenty-first persisted window and accepts its relay retry after draining",
2282 + () => {
2283 + const workers = new Map<string, Effect.Effect<void>>();
2284 + return withService(
2285 + ({ service, sql, launches }) =>
2286 + Effect.gen(function* () {
2287 + const { task } = yield* service.upsert(
2288 + yield* webhookTaskInput({
2289 + prompt: "{{body.group}}",
2290 + schedule: {
2291 + type: "webhook",
2292 + dedupe: {
2293 + replayKey: "{{body.group}}",
2294 + coalesceKey: "{{body.group}}",
2295 + coalesceWindowSeconds: 10,
2296 + },
2297 + },
2298 + }),
2299 + );
2300 + const request = (group: number) => {
2301 + const bodyText = JSON.stringify({ group });
2302 + return requestFor(task, {
2303 + bodyText,
2304 + body: new TextEncoder().encode(bodyText),
2305 + relayDeliveryId: `window-${group}`,
2306 + });
2307 + };
2308 + for (let i = 0; i < 20; i++)
2309 + assert.equal((yield* service.triggerWebhook(request(i)))._tag, "accepted");
2310 + const overflow = yield* service.triggerWebhook(request(20));
2311 + assert.equal(overflow._tag === "rate_limited" && overflow.outcome, "queue_full");
2312 + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 20);
2313 + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`).length, 20);
2314 + assert.equal(
2315 + (yield* sql`SELECT 1 FROM scheduled_task_webhook_relay_deliveries`).length,
2316 + 20,
2317 + );
2318 + yield* TestClock.adjust("10 seconds");
2319 + yield* drainPendingDispatch(workers);
2320 + for (let i = 0; i < 20; i++) yield* Queue.take(launches);
2321 + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0);
2322 + const retry = yield* service.triggerWebhook(request(20));
2323 + assert.equal(retry._tag === "accepted" && retry.outcome, "accepted");
2324 + yield* TestClock.adjust("10 seconds");
2325 + yield* drainPendingDispatch(workers);
2326 + assert.include((yield* Queue.take(launches)).initialMessage!.text, "20");
2327 + assert.equal((yield* service.list()).tasks[0]!.runCount, 21);
2328 + }),
2329 + { workers },
2330 + );
2331 + },
2332 +);
2333 +
2334 +it.effect(
2335 + "recovery finishes failed-window claim cleanup without counting the attempt twice",
2336 + () => {
2337 + const workers = new Map<string, Effect.Effect<void>>();
2338 + return withService(
2339 + ({ service, sql, launches }) =>
2340 + Effect.gen(function* () {
2341 + const { task } = yield* service.upsert(
2342 + yield* webhookTaskInput({
2343 + schedule: {
2344 + type: "webhook",
2345 + dedupe: {
2346 + replayKey: "{{headers.delivery-id}}",
2347 + coalesceKey: "{{body}}",
2348 + coalesceWindowSeconds: 10,
2349 + },
2350 + },
2351 + }),
2352 + );
2353 + const request = (id: string) =>
2354 + requestFor(task, { headers: { "delivery-id": id }, relayDeliveryId: id });
2355 + yield* service.triggerWebhook(request("A"));
2356 + yield* service.triggerWebhook(request("B"));
2357 + yield* sql`CREATE TRIGGER fail_claim_cleanup BEFORE DELETE ON scheduled_task_webhook_relay_deliveries
2358 + BEGIN SELECT RAISE(FAIL, 'cleanup unavailable'); END`;
2359 + yield* TestClock.adjust("10 seconds");
2360 + yield* drainPendingDispatch(workers);
2361 + yield* Queue.take(launches);
2362 + assert.equal((yield* service.list()).tasks[0]!.runCount, 1);
2363 + assert.equal(
2364 + (yield* sql<{ state: string }>`SELECT state FROM scheduled_task_webhook_pending`)[0]!
2365 + .state,
2366 + "failed",
2367 + );
2368 + yield* sql`DROP TRIGGER fail_claim_cleanup`;
2369 + const restartedWorkers = new Map<string, Effect.Effect<void>>();
2370 + yield* withService(
2371 + ({ service: restarted, launches: restartedLaunches }) =>
2372 + Effect.gen(function* () {
2373 + yield* drainPendingDispatch(restartedWorkers);
2374 + assert.equal(yield* Queue.size(restartedLaunches), 0);
2375 + assert.equal((yield* restarted.list()).tasks[0]!.runCount, 1);
2376 + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0);
2377 + assert.equal(
2378 + (yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`).length,
2379 + 0,
2380 + );
2381 + assert.equal(
2382 + (yield* sql`SELECT 1 FROM scheduled_task_webhook_relay_deliveries`).length,
2383 + 0,
2384 + );
2385 + const retry = yield* restarted.triggerWebhook(request("B"));
2386 + assert.equal(retry._tag === "accepted" && retry.outcome, "accepted");
2387 + yield* TestClock.adjust("10 seconds");
2388 + yield* drainPendingDispatch(restartedWorkers);
2389 + yield* Queue.take(restartedLaunches);
2390 + assert.equal((yield* restarted.list()).tasks[0]!.runCount, 2);
2391 + }),
2392 + { sql, workers: restartedWorkers },
2393 + );
2394 + }),
2395 + { workers, failLaunches: { remaining: 1 } },
2396 + );
2397 + },
2398 +);
2399 +
2400 +it.effect("a full run budget still accepts replay hits and the latest open-window payload", () => {
2401 + const workers = new Map<string, Effect.Effect<void>>();
2402 + return withService(
2403 + ({ service, launches, sql }) =>
2404 + Effect.gen(function* () {
2405 + const { task } = yield* service.upsert(
2406 + yield* webhookTaskInput({
2407 + prompt: "{{body.value}}",
2408 + schedule: {
2409 + type: "webhook",
2410 + dedupe: {
2411 + replayKey: "{{headers.delivery-id}}",
2412 + coalesceKey: "{{body.group}}",
2413 + coalesceWindowSeconds: 10,
2414 + },
2415 + },
2416 + }),
2417 + );
2418 + const request = (id: string, value: string, group?: string) => {
2419 + const bodyText = JSON.stringify({ value, ...(group === undefined ? {} : { group }) });
2420 + return requestFor(task, {
2421 + bodyText,
2422 + body: new TextEncoder().encode(bodyText),
2423 + headers: { "delivery-id": id },
2424 + });
2425 + };
2426 + for (let i = 0; i < 59; i++) {
2427 + const result = yield* waitForSpan(
2428 + service.triggerWebhook(request(String(i), String(i))),
2429 + "ScheduledTaskService.runWebhookDelivery",
2430 + );
2431 + assert.equal(result._tag === "accepted" && result.outcome, "accepted");
2432 + yield* Queue.take(launches);
2433 + }
2434 + const opened = yield* service.triggerWebhook(request("B1", "old", "B"));
2435 + assert.equal(opened._tag === "accepted" && opened.outcome, "accepted");
2436 + const duplicate = yield* service.triggerWebhook(request("0", "duplicate"));
2437 + assert.equal(duplicate._tag === "accepted" && duplicate.outcome, "duplicate");
2438 + const update = yield* service.triggerWebhook(request("B2", "latest", "B"));
2439 + assert.equal(update._tag === "accepted" && update.outcome, "coalesced");
2440 + assert.equal(
2441 + (yield* service.triggerWebhook(request("C", "new window", "C")))._tag,
2442 + "rate_limited",
2443 + );
2444 + assert.equal(
2445 + (yield* service.triggerWebhook(request("D", "new immediate")))._tag,
2446 + "rate_limited",
2447 + );
2448 + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 1);
2449 + assert.equal(yield* Queue.size(launches), 0);
2450 + yield* TestClock.adjust("10 seconds");
2451 + yield* drainPendingDispatch(workers);
2452 + assert.include((yield* Queue.take(launches)).initialMessage!.text, "latest");
2453 + assert.equal(yield* Queue.size(launches), 0);
2454 + assert.equal((yield* service.list()).tasks[0]!.runCount, 60);
2455 + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0);
2456 + }),
2457 + { workers },
2458 + );
2459 +});
2460 +
2461 +it.effect("dispatches an overdue window exactly once after its deadline passed while stopped", () =>
2462 + Effect.gen(function* () {
2463 + const sql = yield* SqlClient.SqlClient;
2464 + const task = yield* withService(
2465 + ({ service, launches }) =>
2466 + Effect.gen(function* () {
2467 + const { task } = yield* service.upsert(
2468 + yield* webhookTaskInput({
2469 + schedule: {
2470 + type: "webhook",
2471 + dedupe: { coalesceKey: "{{body}}", coalesceWindowSeconds: 10 },
2472 + },
2473 + }),
2474 + );
2475 + yield* service.triggerWebhook(requestFor(task));
2476 + assert.equal(yield* Queue.size(launches), 0);
2477 + return task;
2478 + }),
2479 + { sql, workers: new Map() },
2480 + );
2481 + // The first service scope is closed before the deadline advances.
2482 + yield* TestClock.adjust("30 seconds");
2483 + const restartedWorkers = new Map<string, Effect.Effect<void>>();
2484 + yield* withService(
2485 + ({ service, launches }) =>
2486 + Effect.gen(function* () {
2487 + yield* drainPendingDispatch(restartedWorkers);
2488 + yield* Queue.take(launches);
2489 + for (let i = 0; i < 3; i++) yield* restartedWorkers.get("webhook-coalescing")!;
2490 + assert.equal(yield* Queue.size(launches), 0);
2491 + assert.equal((yield* service.list()).tasks[0]!.runCount, 1);
2492 + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0);
2493 + }),
2494 + { sql, workers: restartedWorkers },
2495 + );
2496 + const secondRestartWorkers = new Map<string, Effect.Effect<void>>();
2497 + yield* withService(
2498 + ({ service, launches }) =>
2499 + Effect.gen(function* () {
2500 + yield* secondRestartWorkers.get("webhook-coalescing")!;
2501 + assert.equal(yield* Queue.size(launches), 0);
2502 + assert.equal(
2503 + (yield* service.list()).tasks.find((entry) => entry.id === task.id)!.runCount,
2504 + 1,
2505 + );
2506 + }),
2507 + { sql, workers: secondRestartWorkers },
2508 + );
2509 + }).pipe(Effect.provide(SqlitePersistence.layerMemory)),
2510 +);
2511 +
2512 +it.effect("removes a window with a render defect and dispatches the task's next window", () => {
2513 + const workers = new Map<string, Effect.Effect<void>>();
2514 + return withService(
2515 + ({ service, sql, launches }) =>
2516 + Effect.gen(function* () {
2517 + const { task } = yield* service.upsert(
2518 + yield* webhookTaskInput({
2519 + prompt: "{{body.deep}}",
2520 + schedule: {
2521 + type: "webhook",
2522 + dedupe: {
2523 + replayKey: "{{headers.delivery-id}}",
2524 + coalesceKey: "{{headers.group}}",
2525 + coalesceWindowSeconds: 10,
2526 + },
2527 + },
2528 + }),
2529 + );
2530 + const request = (id: string, bodyText: string) =>
2531 + requestFor(task, {
2532 + bodyText,
2533 + body: new TextEncoder().encode(bodyText),
2534 + headers: { "delivery-id": id, group: id },
2535 + relayDeliveryId: id,
2536 + });
2537 + const bad = request(
2538 + "bad-render",
2539 + '{"deep":' + "[".repeat(20000) + "0" + "]".repeat(20000) + "}",
2540 + );
2541 + assert.throws(() => renderWebhookPrompt(task.prompt, bad, "frame"), RangeError);
2542 + const accepted = yield* service.triggerWebhook(bad);
2543 + assert.equal(accepted._tag, "accepted");
2544 + yield* service.triggerWebhook(request("good-render", '{"deep":"next window"}'));
2545 + yield* TestClock.adjust("10 seconds");
2546 + yield* drainPendingDispatch(workers);
2547 + assert.include((yield* Queue.take(launches)).initialMessage!.text, "next window");
2548 + assert.equal(yield* Queue.size(launches), 0);
2549 + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0);
2550 + assert.equal(
2551 + (yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys WHERE delivery_id = 'delivery:relay:bad-render'`)
2552 + .length,
2553 + 0,
2554 + );
2555 + assert.equal(
2556 + (yield* sql`SELECT 1 FROM scheduled_task_webhook_relay_deliveries WHERE relay_delivery_id = 'bad-render'`)
2557 + .length,
2558 + 0,
2559 + );
2560 + const failed = (yield* service.getWebhookDelivery({
2561 + id: task.id,
2562 + deliveryId: ScheduledTaskWebhookDeliveryId.make("delivery:relay:bad-render"),
2563 + acceptAnyOutcome: true,
2564 + })).delivery;
2565 + assert.equal(failed.outcome, "dispatch_failed");
2566 + assert.equal(failed.error, "The pending webhook payload could not be prepared.");
2567 + yield* workers.get("webhook-coalescing")!;
2568 + assert.equal((yield* service.list()).tasks[0]!.runCount, 1);
2569 + }),
2570 + { workers },
2571 + );
2572 +});
2573 +
2574 +it.effect.each([false, true])(
2575 + "counts an interrupted immediate run beside an unrelated running window (corrupt task: %s)",
2576 + (corrupt) => {
2577 + const workers = new Map<string, Effect.Effect<void>>();
2578 + const completedLaunches = new Set<string>();
2579 + return Effect.gen(function* () {
2580 + const gate = yield* Deferred.make<void>();
2581 + yield* withService(
2582 + ({ service, sql, launches }) =>
2583 + Effect.gen(function* () {
2584 + const { task } = yield* service.upsert(
2585 + yield* webhookTaskInput({
2586 + prompt: "{{body.value}}",
2587 + schedule: {
2588 + type: "webhook",
2589 + dedupe: { coalesceKey: "{{headers.group}}", coalesceWindowSeconds: 10 },
2590 + },
2591 + }),
2592 + );
2593 + const window = yield* service.triggerWebhook(
2594 + requestFor(task, { headers: { group: "old" }, bodyText: '{"value":"old"}' }),
2595 + );
2596 + yield* sql`CREATE TRIGGER fail_old_completion BEFORE UPDATE ON scheduled_tasks
2597 + WHEN NEW.last_run_status = 'succeeded' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`;
2598 + yield* TestClock.adjust("10 seconds");
2599 + yield* drainPendingDispatch(workers);
2600 + yield* Queue.take(launches);
2601 + yield* sql`DROP TRIGGER fail_old_completion`;
2602 + assert.equal(
2603 + (yield* sql`SELECT 1 FROM scheduled_task_webhook_pending WHERE state = 'running'`)
2604 + .length,
2605 + 1,
2606 + );
2607 + const immediate = yield* service.triggerWebhook(
2608 + requestFor(task, { bodyText: '{"value":"immediate"}' }),
2609 + );
2610 + yield* Queue.take(launches);
2611 + const [before] = yield* sql<{
2612 + last_run_webhook_delivery_id: string;
2613 + model_selection_json: string;
2614 + run_count: number;
2615 + }>`SELECT last_run_webhook_delivery_id, model_selection_json, run_count FROM scheduled_tasks WHERE task_id = ${task.id}`;
2616 + assert.equal(
2617 + before!.last_run_webhook_delivery_id,
2618 + immediate._tag === "accepted" ? immediate.deliveryId : "",
2619 + );
2620 + assert.notEqual(
2621 + before!.last_run_webhook_delivery_id,
2622 + window._tag === "accepted" ? window.deliveryId : "",
2623 + );
2624 + assert.equal(before!.run_count, 0);
2625 + if (corrupt)
2626 + yield* sql`UPDATE scheduled_tasks SET model_selection_json = 'broken' WHERE task_id = ${task.id}`;
2627 + const restartedWorkers = new Map<string, Effect.Effect<void>>();
2628 + yield* withService(
2629 + ({ service: restarted, launches: recoveredLaunches }) =>
2630 + Effect.gen(function* () {
2631 + const [recovered] = yield* sql<{
2632 + run_count: number;
2633 + last_run_status: string;
2634 + }>`SELECT run_count, last_run_status FROM scheduled_tasks WHERE task_id = ${task.id}`;
2635 + assert.equal(recovered!.run_count, 1);
2636 + assert.equal(recovered!.last_run_status, "failed");
2637 + if (corrupt)
2638 + yield* sql`UPDATE scheduled_tasks SET model_selection_json = ${before!.model_selection_json} WHERE task_id = ${task.id}`;
2639 + yield* drainPendingDispatch(restartedWorkers);
2640 + assert.equal(yield* Queue.size(recoveredLaunches), 0);
2641 + assert.equal((yield* restarted.list()).tasks[0]!.runCount, 2);
2642 + assert.equal(
2643 + (yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length,
2644 + 0,
2645 + );
2646 + yield* restartedWorkers.get("webhook-coalescing")!;
2647 + assert.equal((yield* restarted.list()).tasks[0]!.runCount, 2);
2648 + }),
2649 + { sql, workers: restartedWorkers, completedLaunches },
2650 + );
2651 + }),
2652 + {
2653 + workers,
2654 + completedLaunches,
2655 + gateForTask: (input) =>
2656 + input.initialMessage?.text.includes(">immediate<") ? gate : undefined,
2657 + },
2658 + );
2659 + });
2660 + },
2661 +);
2662 +
2663 +it.effect("checks filters after the signature and max age, and runs only matching deliveries", () =>
2664 + withService(({ service, launches }) =>
2665 + Effect.gen(function* () {
2666 + const { task } = yield* service.upsert(
2667 + yield* webhookTaskInput({
2668 + schedule: {
2669 + type: "webhook",
2670 + signature: {
2671 + header: "x-hub-signature-256",
2672 + encoding: "hex",
2673 + prefix: "sha256=",
2674 + secret: "s3cret",
2675 + },
2676 + maxDeliveryAgeMinutes: 30,
2677 + filters: [
2678 + { path: "headers.x-github-event", op: "equals", value: "pull_request" },
2679 + { path: "body.action", op: "in", value: ["opened", "reopened"] },
2680 + ],
2681 + },
2682 + }),
2683 + );
2684 + const delivery = (
2685 + event: string,
2686 + action: string,
2687 + options: { readonly signed?: boolean; readonly receivedAt?: string } = {},
2688 + ) => {
2689 + const body = new TextEncoder().encode(
2690 + JSON.stringify({ action, pull_request: { url: "https://github.com/org/repo/pull/45" } }),
2691 + );
2692 + const signature = `sha256=${NodeCrypto.createHmac("sha256", "s3cret").update(body).digest("hex")}`;
2693 + return requestFor(task, {
2694 + headers: {
2695 + "content-type": "application/json",
2696 + "x-github-event": event,
2697 + ...(options.signed === false ? {} : { "x-hub-signature-256": signature }),
2698 + },
2699 + body,
2700 + bodyText: new TextDecoder().decode(body),
2701 + ...(options.receivedAt === undefined
2702 + ? {}
2703 + : { relayDeliveryId: "old", receivedAt: options.receivedAt }),
2704 + });
2705 + };
2706 + const now = yield* DateTime.now;
2707 + const old = DateTime.formatIso(DateTime.subtract(now, { minutes: 31 }));
2708 +
2709 + // A request failing a filter is still refused first for its signature or age.
2710 + const unsigned = yield* service.triggerWebhook(delivery("push", "opened", { signed: false }));
2711 + assert.equal(unsigned._tag, "rejected_signature");
2712 + const tooOld = yield* service.triggerWebhook(delivery("push", "opened", { receivedAt: old }));
2713 + assert.equal(tooOld._tag, "expired");
2714 +
2715 + const wrongEvent = yield* service.triggerWebhook(delivery("push", "opened"));
2716 + const wrongAction = yield* service.triggerWebhook(delivery("pull_request", "closed"));
2717 + assert.deepInclude(wrongEvent, { _tag: "accepted", outcome: "filtered" });
2718 + assert.deepInclude(wrongAction, { _tag: "accepted", outcome: "filtered" });
2719 + assert.equal(yield* Queue.size(launches), 0);
2720 + const [listed] = (yield* service.list()).tasks;
2721 + assert.equal(listed?.runCount, 0);
2722 + assert.equal(listed?.lastRunStatus, "never");
2723 +
2724 + const matched = yield* service.triggerWebhook(delivery("pull_request", "opened"));
2725 + assert.deepInclude(matched, { _tag: "accepted", outcome: "accepted" });
2726 + assert.include((yield* Queue.take(launches)).initialMessage?.text ?? "", "pull/45");
2727 +
2728 + const deliveries = (yield* service.listWebhookDeliveries({
2729 + id: task.id,
2730 + acceptAnyOutcome: true,
2731 + })).deliveries;
2732 + const filtered = deliveries
2733 + .filter((entry) => entry.outcome === "filtered")
2734 + .map((entry) => [entry.signatureVerified, entry.error]);
2735 + assert.sameDeepMembers(filtered, [
2736 + [true, "Filter 1 (headers.x-github-event) did not match."],
2737 + [true, "Filter 2 (body.action) did not match."],
2738 + ]);
2739 + }),
2740 + ),
2741 +);
2742 +
2743 +const pullRequestFilterInput = () =>
2744 + webhookTaskInput({
2745 + schedule: {
2746 + type: "webhook",
2747 + filters: [{ path: "headers.x-github-event", op: "equals", value: "pull_request" }],
2748 + },
2749 + });
2750 +const githubEvent = (event: string) => ({
2751 + headers: { "content-type": "application/json", "x-github-event": event },
2752 +});
2753 +
2754 +it.effect("filtered deliveries take no run slot and log at most 10 rows a minute", () =>
2755 + withService(({ service, launches }) =>
2756 + Effect.gen(function* () {
2757 + const { task } = yield* service.upsert(yield* pullRequestFilterInput());
2758 + const ignored = yield* Effect.forEach(Array.from({ length: 70 }), () =>
2759 + service.triggerWebhook(requestFor(task, githubEvent("push"))),
2760 + );
2761 + assert.isTrue(
2762 ignored.every((result) => result._tag === "accepted" && result.outcome === "filtered"),
2763 );
2764 const matched = yield* service.triggerWebhook(requestFor(task, githubEvent("pull_request")));
@@ -1157,3 +2863,75 @@ it.effect("leaves filtered rows out of the list for clients that only decode old
2863 }),
2864 ),
2865 );
2866 +
2867 +it.effect("filters run before replay and coalescing without claiming ignored deliveries", () => {
2868 + const workers = new Map<string, Effect.Effect<void>>();
2869 + return withService(
2870 + ({ service, launches, sql }) =>
2871 + Effect.gen(function* () {
2872 + const { task } = yield* service.upsert(
2873 + yield* webhookTaskInput({
2874 + prompt: "{{body.value}}",
2875 + schedule: {
2876 + type: "webhook",
2877 + filters: [{ path: "body.run", op: "equals", value: "true" }],
2878 + dedupe: {
2879 + replayKey: "{{headers.delivery-id}}",
2880 + coalesceKey: "{{body.group}}",
2881 + coalesceWindowSeconds: 10,
2882 + },
2883 + },
2884 + }),
2885 + );
2886 + const request = (id: string, run: boolean, value: string) => {
2887 + const bodyText = JSON.stringify({ run, group: "B", value });
2888 + return requestFor(task, {
2889 + headers: { "delivery-id": id },
2890 + bodyText,
2891 + body: new TextEncoder().encode(bodyText),
2892 + relayDeliveryId: id,
2893 + });
2894 + };
2895 + const ignored = yield* service.triggerWebhook(request("A", false, "ignored"));
2896 + assert.deepInclude(ignored, { _tag: "accepted", outcome: "filtered" });
2897 + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`).length, 0);
2898 + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0);
2899 + assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_relay_deliveries`).length, 0);
2900 + const filteredId = ignored._tag === "accepted" ? ignored.deliveryId : assert.fail();
2901 + const oldClient = yield* service
2902 + .getWebhookDelivery({ id: task.id, deliveryId: filteredId })
2903 + .pipe(Effect.result);
2904 + assert.equal(oldClient._tag, "Failure");
2905 + const currentClient = yield* service.getWebhookDelivery({
2906 + id: task.id,
2907 + deliveryId: filteredId,
2908 + acceptAnyOutcome: true,
2909 + });
2910 + assert.equal(currentClient.delivery.outcome, "filtered");
2911 + assert.deepInclude(yield* service.triggerWebhook(request("A", true, "first")), {
2912 + _tag: "accepted",
2913 + outcome: "accepted",
2914 + });
2915 + assert.deepInclude(
2916 + yield* service.triggerWebhook({
2917 + ...request("A", false, "ignored replay"),
2918 + relayDeliveryId: "ignored-replay",
2919 + }),
2920 + { _tag: "accepted", outcome: "filtered" },
2921 + );
2922 + assert.deepInclude(yield* service.triggerWebhook(request("B", false, "ignored update")), {
2923 + _tag: "accepted",
2924 + outcome: "filtered",
2925 + });
2926 + assert.deepInclude(yield* service.triggerWebhook(request("B", true, "latest")), {
2927 + _tag: "accepted",
2928 + outcome: "coalesced",
2929 + });
2930 + yield* TestClock.adjust("10 seconds");
2931 + yield* drainPendingDispatch(workers);
2932 + assert.include((yield* Queue.take(launches)).initialMessage!.text, "latest");
2933 + assert.equal(yield* Queue.size(launches), 0);
2934 + }),
2935 + { workers },
2936 + );
2937 +});
apps/server/src/scheduledTasks/webhookDedupe.ts new
+128
@@ -0,0 +1,128 @@
1 +// @effect-diagnostics-next-line nodeBuiltinImport:off -- Keys need a deterministic digest, not random bytes.
2 +import * as NodeCrypto from "node:crypto";
3 +import { ScheduledTask, ScheduledTaskError } from "@t3tools/contracts";
4 +import * as Effect from "effect/Effect";
5 +import * as Schema from "effect/Schema";
6 +import * as SqlClient from "effect/sql/SqlClient";
7 +import { renderWebhookPrompt, type WebhookRequest } from "./webhookTemplate.ts";
8 +
9 +function key(template: string | undefined, request: WebhookRequest): string {
10 + if (template === undefined) return "";
11 + const rendered = renderWebhookPrompt(template, request, "", true);
12 + return rendered.missing.length > 0 || rendered.prompt === ""
13 + ? ""
14 + : NodeCrypto.createHash("sha256").update(rendered.prompt).digest("hex");
15 +}
16 +
17 +export const releaseWebhookReplay = Effect.fn("ScheduledTaskService.releaseWebhookReplay")(
18 + function* (deliveryId: string) {
19 + const sql = yield* SqlClient.SqlClient;
20 + yield* sql`DELETE FROM scheduled_task_webhook_replay_keys WHERE delivery_id = ${deliveryId}`;
21 + },
22 +);
23 +
24 +const hasReplayKey = Effect.fn(function* (taskId: string, replayKey: string, now: number) {
25 + const sql = yield* SqlClient.SqlClient;
26 + return (
27 + replayKey !== "" &&
28 + (yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys
29 + WHERE task_id = ${taskId} AND replay_key = ${replayKey} AND expires_at > ${now}`).length > 0
30 + );
31 +});
32 +const claimReplayKey = Effect.fn(function* (
33 + taskId: string,
34 + replayKey: string,
35 + deliveryId: string,
36 + now: number,
37 +) {
38 + if (replayKey === "") return;
39 + const sql = yield* SqlClient.SqlClient;
40 + yield* sql`DELETE FROM scheduled_task_webhook_replay_keys WHERE expires_at <= ${now}`;
41 + yield* sql`INSERT INTO scheduled_task_webhook_replay_keys (task_id, replay_key, delivery_id, expires_at)
42 + VALUES (${taskId}, ${replayKey}, ${deliveryId}, ${now + 7 * 24 * 60 * 60 * 1000})`;
43 +});
44 +
45 +export const acceptWebhookDedupe = Effect.fn("ScheduledTaskService.acceptWebhookDedupe")(function* (
46 + task: ScheduledTask,
47 + request: WebhookRequest,
48 + deliveryId: string,
49 + now: number,
50 + reserve: Effect.Effect<void, "first_rejected" | "rejected">,
51 + reserveCoalescedLog: Effect.Effect<boolean>,
52 + checkCapacity: Effect.Effect<void, "queue_full" | ScheduledTaskError>,
53 + replaceDeliveryRequest: (ownerId: string) => Effect.Effect<void, ScheduledTaskError>,
54 +) {
55 + const sql = yield* SqlClient.SqlClient;
56 + const dedupe = task.schedule.type === "webhook" ? task.schedule.dedupe : undefined;
57 + if (dedupe === undefined) return "run" as const;
58 + const replayKey = key(dedupe.replayKey, request);
59 + if (yield* hasReplayKey(task.id, replayKey, now)) return "duplicate" as const;
60 + const coalesceKey = key(dedupe.coalesceKey, request);
61 + if (coalesceKey === "") return "run" as const;
62 + const pending = yield* sql<{
63 + delivery_id: string;
64 + }>`SELECT delivery_id FROM scheduled_task_webhook_pending
65 + WHERE task_id = ${task.id} AND coalesce_key = ${coalesceKey} AND state = 'pending' AND due_at > ${now}`;
66 + const logCoalesced = pending.length > 0 ? yield* reserveCoalescedLog : true;
67 + if (pending.length === 0) {
68 + yield* checkCapacity;
69 + yield* reserve;
70 + }
71 + // Coalesced requests retain their validation slot, bounding writes without dropping accepted payloads.
72 + yield* claimReplayKey(task.id, replayKey, pending[0]?.delivery_id ?? deliveryId, now);
73 + const requestJson = yield* encodePendingRequest({
74 + method: request.method,
75 + path: request.path,
76 + query: request.query,
77 + headers: request.headers,
78 + bodyText: request.bodyText,
79 + });
80 + if (pending.length > 0) {
81 + yield* sql`UPDATE scheduled_task_webhook_pending SET request_json = ${requestJson},
82 + delivery_ids_json = json_insert(delivery_ids_json, '$[#]', ${deliveryId}) WHERE delivery_id = ${pending[0]!.delivery_id}`;
83 + yield* replaceDeliveryRequest(pending[0]!.delivery_id);
84 + return logCoalesced ? ("coalesced" as const) : ("coalesced_unlogged" as const);
85 + }
86 + // Expired windows are immutable, even if the scheduler has not dispatched them yet.
87 + yield* sql`UPDATE scheduled_task_webhook_pending SET state = 'dispatching'
88 + WHERE task_id = ${task.id} AND coalesce_key = ${coalesceKey} AND state = 'pending' AND due_at <= ${now}`;
89 + yield* sql`INSERT INTO scheduled_task_webhook_pending (delivery_id, task_id, coalesce_key, due_at, task_json, request_json, delivery_ids_json)
90 + VALUES (${deliveryId}, ${task.id}, ${coalesceKey}, ${now + (dedupe.coalesceWindowSeconds ?? 120) * 1000}, ${yield* encodePendingTask(task)}, ${requestJson}, json_array(${deliveryId}))`;
91 + return "pending" as const;
92 +});
93 +
94 +export const acceptImmediateWebhookDedupe = Effect.fn(
95 + "ScheduledTaskService.acceptImmediateWebhookDedupe",
96 +)(function* (
97 + task: ScheduledTask,
98 + request: WebhookRequest,
99 + deliveryId: string,
100 + now: number,
101 + reserve: Effect.Effect<void, "first_rejected" | "rejected" | "queue_full" | ScheduledTaskError>,
102 +) {
103 + const replayKey = key(
104 + task.schedule.type === "webhook" ? task.schedule.dedupe?.replayKey : undefined,
105 + request,
106 + );
107 + if (yield* hasReplayKey(task.id, replayKey, now)) return "duplicate" as const;
108 + yield* reserve;
109 + yield* claimReplayKey(task.id, replayKey, deliveryId, now);
110 + return "run" as const;
111 +});
112 +
113 +const PendingTaskJson = Schema.fromJsonString(ScheduledTask);
114 +const encodePendingTask = Schema.encodeEffect(PendingTaskJson);
115 +export const decodePendingTask = Schema.decodeUnknownEffect(PendingTaskJson);
116 +const PendingRequestJson = Schema.fromJsonString(
117 + Schema.Struct({
118 + method: Schema.String,
119 + path: Schema.String,
120 + query: Schema.String,
121 + headers: Schema.Record(Schema.String, Schema.String),
122 + bodyText: Schema.String,
123 + }),
124 +);
125 +const encodePendingRequest = Schema.encodeEffect(PendingRequestJson);
126 +export const decodePendingRequest = Schema.decodeUnknownEffect(PendingRequestJson);
127 +export const dedupeError = (cause: unknown) =>
128 + new ScheduledTaskError({ message: "Could not persist webhook dedupe state.", cause });
apps/server/src/scheduledTasks/webhookRoute.test.ts
+1
@@ -236,6 +236,7 @@ describe("webhook route", () => {
236 const response = await handler(post("/api/hooks/id/tok", "{}"));
237 expect(response.status).toBe(status);
238 expect(response.headers.get("x-t3-hook-outcome")).toBe(outcome);
239 + if (status === 429) expect(await response.json()).toEqual({ error: outcome });
240 } finally {
241 await dispose();
242 }
apps/server/src/scheduledTasks/webhookRoute.ts
+1 -1
@@ -118,7 +118,7 @@ export const layer = HttpApiBuilder.group(
118 case "disabled":
119 return json(409, { error: "hook_disabled" }, "disabled");
120 case "rate_limited":
121 - return json(429, { error: "rate_limited" }, result.outcome);
121 + return json(429, { error: result.outcome }, result.outcome);
122 case "expired":
123 return json(410, { error: "delivery_too_old" }, "expired");
124 case "error":
apps/server/src/scheduledTasks/webhookTemplate.ts
+3 -1
@@ -150,6 +150,7 @@ export function renderWebhookPrompt(
150 template: string,
151 request: WebhookRequest,
152 frameId: string,
153 + raw = false,
154 ): RenderedWebhookPrompt {
155 const missing: string[] = [];
156 const resolve = webhookExpressionResolver(request);
@@ -160,12 +161,13 @@ export function renderWebhookPrompt(
161 const expression = rawExpression.trim();
162 const value = resolve(expression);
163 const text = value === undefined || value === null ? undefined : stringify(value);
163 - if (text === undefined) {
164 + if (text === undefined || (raw && text === "")) {
165 if (!missing.includes(expression)) missing.push(expression);
166 return "";
167 }
168 // An empty value carries no request data, so it needs no frame or note.
169 if (text === "") return "";
170 + if (raw) return text;
171 framed = true;
172 return `<${tag}>${text.replace(MENTION, "$&\u2060")}</${tag}>`;
173 });
apps/web/src/components/settings/ScheduledTasksSettings.tsx
+87 -30
@@ -22,7 +22,6 @@ import type {
22 ScheduledTaskId,
23 ScheduledTaskSchedule,
24 ScheduledTaskUpsertInput,
25 - ScheduledTaskWebhookDeliveryOutcome,
25 ScheduledTaskWebhookDeliverySummary,
26 ThreadId,
27 } from "@t3tools/contracts";
@@ -81,6 +80,8 @@ import { SettingsGroup } from "./SettingsGroup";
80 import { settingsScopeEnvironmentLabel, settingsScopeEnvironmentLabels } from "./settingsScopeAxis";
81 import {
82 WEBHOOK_SIGNATURE_DEFAULTS,
83 + webhookDraftErrors,
84 + webhookDeliveryLabel,
85 matchesScheduledTaskScope,
86 scheduleFromDraft,
87 scheduledTaskDefaultModel,
@@ -152,6 +153,9 @@ const EMPTY_DRAFT: DraftState = {
153 ...WEBHOOK_SIGNATURE_DEFAULTS,
154 signatureSecret: "",
155 maxDeliveryAgeMinutes: "",
156 + replayKey: "",
157 + coalesceKey: "",
158 + coalesceWindowSeconds: "",
159 filters: [],
160 savedFilters: [],
161 };
@@ -328,26 +332,15 @@ export function ScheduledRelativeLabel(props: { value: string | null }) {
332 return relativeLabel(props.value, Date.parse(`${nowMinute}:00.000Z`));
333 }
334
331 -const DELIVERY_OUTCOME_LABELS: Record<ScheduledTaskWebhookDeliveryOutcome, string> = {
332 - accepted: "Ran",
333 - dispatch_failed: "Run failed",
334 - rejected_signature: "Bad signature",
335 - disabled: "Task paused",
336 - rate_limited: "Rate limited",
337 - expired: "Too old",
338 - filtered: "Filtered",
339 -};
340 -
341 -// A newer server can report an outcome this client has no label for.
342 -function deliveryOutcomeLabel(outcome: string): string {
343 - return Object.hasOwn(DELIVERY_OUTCOME_LABELS, outcome)
344 - ? DELIVERY_OUTCOME_LABELS[outcome as ScheduledTaskWebhookDeliveryOutcome]
345 - : outcome.replaceAll("_", " ");
346 -}
347 -
335 function deliveryOutcomeVariant(outcome: string) {
336 if (outcome === "accepted") return "success";
350 - if (outcome === "disabled" || outcome === "rate_limited" || outcome === "expired") {
337 + if (
338 + outcome === "duplicate" ||
339 + outcome === "coalesced" ||
340 + outcome === "disabled" ||
341 + outcome === "rate_limited" ||
342 + outcome === "expired"
343 + ) {
344 return "warning";
345 }
346 if (outcome === "dispatch_failed" || outcome === "rejected_signature") return "error";
@@ -822,7 +815,7 @@ function WebhookDeliveriesDialog({
815 ? null
816 : serverEnvironment.scheduledTaskWebhookDelivery({
817 environmentId,
825 - input: { id: task.id, deliveryId: selectedId },
818 + input: { id: task.id, deliveryId: selectedId, acceptAnyOutcome: true },
819 }),
820 );
821 const deliveries = deliveriesQuery.data?.deliveries ?? null;
@@ -850,8 +843,12 @@ function WebhookDeliveriesDialog({
843 ) : selected ? (
844 <div className="space-y-4">
845 <div className="flex flex-wrap items-center gap-2 text-sm">
853 - <Badge variant={deliveryOutcomeVariant(selected.outcome)}>
854 - {deliveryOutcomeLabel(selected.outcome)}
846 + <Badge
847 + variant={deliveryOutcomeVariant(
848 + selected.pendingWindow ? "coalesced" : selected.outcome,
849 + )}
850 + >
851 + {webhookDeliveryLabel(selected)}
852 </Badge>
853 <span>
854 {selected.method} · <ScheduledRelativeLabel value={selected.receivedAt} />
@@ -871,8 +868,13 @@ function WebhookDeliveriesDialog({
868 {selected.error}
869 </p>
870 ) : null}
874 - <DeliveryBlock title="Prompt sent to the agent">
875 - {selected.renderedPrompt ?? "No run was started for this request."}
871 + <DeliveryBlock
872 + title={selected.pendingWindow ? "Scheduled run" : "Prompt sent to the agent"}
873 + >
874 + {selected.renderedPrompt ??
875 + (selected.pendingWindow
876 + ? "Run scheduled when the window closes; no run started yet."
877 + : "No run was started for this request.")}
878 </DeliveryBlock>
879 {selected.missingFields.length > 0 ? (
880 <p className="text-sm text-muted-foreground">
@@ -906,8 +908,12 @@ function WebhookDeliveriesDialog({
908 className="flex w-full flex-wrap items-center gap-2 py-2 text-left text-sm hover:bg-accent/40"
909 onClick={() => setSelectedId(delivery.id)}
910 >
909 - <Badge variant={deliveryOutcomeVariant(delivery.outcome)}>
910 - {deliveryOutcomeLabel(delivery.outcome)}
911 + <Badge
912 + variant={deliveryOutcomeVariant(
913 + delivery.pendingWindow ? "coalesced" : delivery.outcome,
914 + )}
915 + >
916 + {webhookDeliveryLabel(delivery)}
917 </Badge>
918 <span>{delivery.method}</span>
919 <span className="text-muted-foreground">
@@ -1193,10 +1199,7 @@ function ScheduledTaskEditorForm({
1199 }
1200 const schedule = scheduleFromDraft(draft);
1201 if (schedule === null) {
1196 - reportFailure(
1197 - "Invalid age limit",
1198 - `Enter whole minutes from 1 to ${MAX_WEBHOOK_DELIVERY_AGE_MINUTES}, or leave it blank.`,
1199 - );
1202 + for (const error of webhookDraftErrors(draft)) reportFailure(error.title, error.description);
1203 return;
1204 }
1205 if (
@@ -1543,6 +1546,60 @@ function ScheduledTaskEditorForm({
1546 filters={draft.filters}
1547 onChange={(filters) => setDraft((current) => ({ ...current, filters }))}
1548 />
1549 + <Field
1550 + label="Replay key"
1551 + hint="template, optional"
1552 + htmlFor="scheduled-task-replay-key"
1553 + >
1554 + <Input
1555 + id="scheduled-task-replay-key"
1556 + placeholder="{{headers.delivery-id}}"
1557 + value={draft.replayKey}
1558 + onChange={(event) =>
1559 + setDraft((current) => ({ ...current, replayKey: event.target.value }))
1560 + }
1561 + />
1562 + </Field>
1563 + <Field
1564 + label="Coalesce key"
1565 + hint="template, optional"
1566 + htmlFor="scheduled-task-coalesce-key"
1567 + >
1568 + <Input
1569 + id="scheduled-task-coalesce-key"
1570 + placeholder="{{body.id}}"
1571 + value={draft.coalesceKey}
1572 + onChange={(event) =>
1573 + setDraft((current) => ({ ...current, coalesceKey: event.target.value }))
1574 + }
1575 + />
1576 + </Field>
1577 + <Field
1578 + label="Coalesce window"
1579 + hint="seconds, 10–3600"
1580 + htmlFor="scheduled-task-coalesce-window"
1581 + >
1582 + <Input
1583 + id="scheduled-task-coalesce-window"
1584 + type="number"
1585 + nativeInput
1586 + min={10}
1587 + max={3600}
1588 + placeholder="120"
1589 + value={draft.coalesceWindowSeconds}
1590 + onChange={(event) =>
1591 + setDraft((current) => ({
1592 + ...current,
1593 + coalesceWindowSeconds: event.target.value,
1594 + }))
1595 + }
1596 + />
1597 + </Field>
1598 + <p className="text-xs text-muted-foreground">
1599 + Replay keys suppress repeat deliveries for 7 days. Coalescing runs the latest
1600 + payload once when the window from the first delivery closes. Empty keys run
1601 + normally.
1602 + </p>
1603 <div className="flex items-center justify-between gap-4">
1604 <div className="min-w-0 space-y-1">
1605 <Label htmlFor="scheduled-task-signature">Require signature</Label>
apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts
+98
@@ -20,6 +20,8 @@ import {
20 scheduledTaskDefaultModel,
21 matchesScheduledTaskScope,
22 scheduleFromDraft,
23 + webhookDraftErrors,
24 + webhookDeliveryLabel,
25 taskToDraft,
26 } from "./scheduledTasksSettings.logic";
27
@@ -175,6 +177,43 @@ describe("webhook scheduled tasks", () => {
177 webhook: { path: "/api/hooks/legacy-task/token", url: null, hasSecret: true },
178 };
179
180 + it("round-trips dedupe settings, clears them, and rejects invalid windows", () => {
181 + const dedupe = {
182 + replayKey: "{{headers.delivery-id}}",
183 + coalesceKey: "{{body.id}}",
184 + coalesceWindowSeconds: 10,
185 + };
186 + const draft = taskToDraft({
187 + ...webhookTask,
188 + schedule: { type: "webhook", signature: null, dedupe },
189 + });
190 + expect(scheduleFromDraft(draft)).toMatchObject({ dedupe });
191 + expect(
192 + scheduleFromDraft({
193 + ...draft,
194 + replayKey: " \n ",
195 + coalesceKey: " \t ",
196 + coalesceWindowSeconds: "",
197 + }),
198 + ).toMatchObject({ dedupe: {} });
199 + expect(
200 + scheduleFromDraft({
201 + ...draft,
202 + replayKey: ` ${dedupe.replayKey} `,
203 + coalesceKey: `\t${dedupe.coalesceKey}\n`,
204 + }),
205 + ).toMatchObject({ dedupe });
206 + expect(
207 + scheduleFromDraft({ ...draft, replayKey: "", coalesceKey: "", coalesceWindowSeconds: "" }),
208 + ).toMatchObject({ dedupe: {} });
209 + expect(scheduleFromDraft({ ...draft, coalesceWindowSeconds: "" })).toMatchObject({
210 + dedupe: { replayKey: dedupe.replayKey, coalesceKey: dedupe.coalesceKey },
211 + });
212 + for (const coalesceWindowSeconds of ["9", "3601", "1.5", "abc"]) {
213 + expect(scheduleFromDraft({ ...draft, coalesceWindowSeconds })).toBeNull();
214 + }
215 + });
216 +
217 it("keeps a stored secret when the secret field is left blank", () => {
218 const draft = taskToDraft(webhookTask);
219 expect(draft.scheduleMode).toBe("webhook");
@@ -183,11 +222,13 @@ describe("webhook scheduled tasks", () => {
222 type: "webhook",
223 signature,
224 maxDeliveryAgeMinutes: null,
225 + dedupe: {},
226 });
227 expect(scheduleFromDraft({ ...draft, signatureSecret: " new " })).toEqual({
228 type: "webhook",
229 signature: { ...signature, secret: "new" },
230 maxDeliveryAgeMinutes: null,
231 + dedupe: {},
232 });
233 });
234
@@ -199,6 +240,7 @@ describe("webhook scheduled tasks", () => {
240 type: "webhook",
241 signature: null,
242 maxDeliveryAgeMinutes: null,
243 + dedupe: {},
244 });
245 });
246
@@ -212,6 +254,7 @@ describe("webhook scheduled tasks", () => {
254 for (const blank of ["", " "]) {
255 expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: blank })).toMatchObject({
256 maxDeliveryAgeMinutes: null,
257 + dedupe: {},
258 });
259 }
260 for (const invalid of ["0", "-5", "1.5", "abc", "1441"]) {
@@ -339,3 +382,58 @@ describe("scheduled task model defaults", () => {
382 ).toBeNull();
383 });
384 });
385 +
386 +it("rejects dedupe templates without a request placeholder", () => {
387 + const draft = taskToDraft({ ...legacyTask, schedule: { type: "webhook", signature: null } });
388 + expect(scheduleFromDraft({ ...draft, replayKey: "literal" })).toBeNull();
389 + expect(scheduleFromDraft({ ...draft, coalesceKey: "{{body.bad" })).toBeNull();
390 +});
391 +
392 +describe("webhook delivery labels and validation messages", () => {
393 + it("preserves Ran for legacy and dispatched deliveries and distinguishes pending windows", () => {
394 + expect(webhookDeliveryLabel({ outcome: "accepted" })).toBe("Ran");
395 + expect(webhookDeliveryLabel({ outcome: "accepted", pendingWindow: false })).toBe("Ran");
396 + expect(webhookDeliveryLabel({ outcome: "accepted", pendingWindow: true })).toBe(
397 + "Opened window",
398 + );
399 + });
400 +
401 + it("keeps the existing age-limit title and body", () => {
402 + const draft = taskToDraft({ ...legacyTask, schedule: { type: "webhook", signature: null } });
403 + expect(
404 + webhookDraftErrors({ ...draft, maxDeliveryAgeMinutes: "0", replayKey: "literal" }),
405 + ).toEqual([
406 + {
407 + title: "Invalid age limit",
408 + description: "Enter whole minutes from 1 to 1440, or leave it blank.",
409 + },
410 + ]);
411 + });
412 +
413 + it.each([
414 + [{ replayKey: "literal" }, "Replay key must contain a valid request placeholder."],
415 + [{ coalesceKey: "{{body.bad" }, "Coalesce key must contain a valid request placeholder."],
416 + [{ coalesceWindowSeconds: "9" }, "Coalesce window must be whole seconds from 10 to 3600."],
417 + ])("names only the invalid webhook field for %o", (invalid, description) => {
418 + const draft = taskToDraft({ ...legacyTask, schedule: { type: "webhook", signature: null } });
419 + expect(webhookDraftErrors({ ...draft, ...invalid })).toEqual([
420 + { title: "Invalid webhook settings", description },
421 + ]);
422 + });
423 +
424 + it("reports separate short errors when several dedupe fields fail", () => {
425 + const draft = taskToDraft({ ...legacyTask, schedule: { type: "webhook", signature: null } });
426 + expect(
427 + webhookDraftErrors({
428 + ...draft,
429 + replayKey: "bad",
430 + coalesceKey: "bad",
431 + coalesceWindowSeconds: "0",
432 + }).map((error) => error.description),
433 + ).toEqual([
434 + "Replay key must contain a valid request placeholder.",
435 + "Coalesce key must contain a valid request placeholder.",
436 + "Coalesce window must be whole seconds from 10 to 3600.",
437 + ]);
438 + });
439 +});
apps/web/src/components/settings/scheduledTasksSettings.logic.ts
+73
@@ -1,5 +1,7 @@
1 +import { isWebhookDedupeTemplate } from "@t3tools/contracts";
2 import {
3 EnvironmentId,
4 + MAX_WEBHOOK_DELIVERY_AGE_MINUTES,
5 type ProjectId,
6 ScheduledTaskId,
7 type ScheduledTask,
@@ -83,6 +85,9 @@ export interface DraftState {
85 /** Write-only: empty keeps the secret already stored on the server. */
86 readonly signatureSecret: string;
87 readonly maxDeliveryAgeMinutes: string;
88 + readonly replayKey: string;
89 + readonly coalesceKey: string;
90 + readonly coalesceWindowSeconds: string;
91 readonly filters: ReadonlyArray<WebhookFilterDraft>;
92 /** The filter rows the form opened with. */
93 readonly savedFilters: ReadonlyArray<WebhookFilterDraft>;
@@ -94,9 +99,62 @@ export const WEBHOOK_SIGNATURE_DEFAULTS = {
99 signaturePrefix: "sha256=",
100 } as const;
101
102 +export function webhookDraftErrors(draft: DraftState) {
103 + if (draft.scheduleMode !== "webhook") return [];
104 + if (parseMaxDeliveryAge(draft.maxDeliveryAgeMinutes) === undefined)
105 + return [
106 + {
107 + title: "Invalid age limit",
108 + description: `Enter whole minutes from 1 to ${MAX_WEBHOOK_DELIVERY_AGE_MINUTES}, or leave it blank.`,
109 + },
110 + ];
111 + const errors: Array<{ title: string; description: string }> = [];
112 + for (const [label, value] of [
113 + ["Replay key", draft.replayKey],
114 + ["Coalesce key", draft.coalesceKey],
115 + ] as const) {
116 + if (value.trim() && !isWebhookDedupeTemplate(value.trim()))
117 + errors.push({
118 + title: "Invalid webhook settings",
119 + description: `${label} must contain a valid request placeholder.`,
120 + });
121 + }
122 + const window = draft.coalesceWindowSeconds.trim();
123 + if (window && (!Number.isInteger(Number(window)) || Number(window) < 10 || Number(window) > 3600))
124 + errors.push({
125 + title: "Invalid webhook settings",
126 + description: "Coalesce window must be whole seconds from 10 to 3600.",
127 + });
128 + return errors;
129 +}
130 +
131 +export function webhookDeliveryLabel(delivery: {
132 + readonly outcome: string;
133 + readonly pendingWindow?: boolean | undefined;
134 +}) {
135 + if (delivery.outcome === "accepted") return delivery.pendingWindow ? "Opened window" : "Ran";
136 + const labels: Record<string, string> = {
137 + dispatch_failed: "Run failed",
138 + duplicate: "Duplicate",
139 + coalesced: "Coalesced",
140 + filtered: "Filtered",
141 + rejected_signature: "Bad signature",
142 + disabled: "Task paused",
143 + rate_limited: "Rate limited",
144 + expired: "Too old",
145 + };
146 + return labels[delivery.outcome] ?? delivery.outcome.replaceAll("_", " ");
147 +}
148 +
149 export function scheduleFromDraft(draft: DraftState): ScheduledTaskUpsertSchedule | null {
150 if (draft.scheduleMode === "webhook") {
151 + if (webhookDraftErrors(draft).length > 0) return null;
152 const maxDeliveryAgeMinutes = parseMaxDeliveryAge(draft.maxDeliveryAgeMinutes);
153 + const replayKey = draft.replayKey.trim();
154 + const coalesceKey = draft.coalesceKey.trim();
155 + const coalesceWindowSeconds = draft.coalesceWindowSeconds.trim()
156 + ? Number(draft.coalesceWindowSeconds)
157 + : undefined;
158 if (maxDeliveryAgeMinutes === undefined) return null;
159 const parsedFilters = parseWebhookFilterDrafts(draft.filters);
160 if (!("filters" in parsedFilters)) return null;
@@ -112,6 +170,15 @@ export function scheduleFromDraft(draft: DraftState): ScheduledTaskUpsertSchedul
170 }
171 : null,
172 maxDeliveryAgeMinutes,
173 + ...(replayKey || coalesceKey || coalesceWindowSeconds !== undefined
174 + ? {
175 + dedupe: {
176 + ...(replayKey ? { replayKey } : {}),
177 + ...(coalesceKey ? { coalesceKey } : {}),
178 + ...(coalesceWindowSeconds === undefined ? {} : { coalesceWindowSeconds }),
179 + },
180 + }
181 + : { dedupe: {} }),
182 // Only edited filters are sent; the server keeps the stored ones otherwise.
183 ...(webhookFilterDraftsChanged(draft.savedFilters, draft.filters)
184 ? { filters: parsedFilters.filters }
@@ -173,6 +240,12 @@ export function taskToDraft(task: ScheduledTask): DraftState {
240 }
241 : { signatureEnabled: false, ...WEBHOOK_SIGNATURE_DEFAULTS }),
242 signatureSecret: "",
243 + replayKey: schedule.type === "webhook" ? (schedule.dedupe?.replayKey ?? "") : "",
244 + coalesceKey: schedule.type === "webhook" ? (schedule.dedupe?.coalesceKey ?? "") : "",
245 + coalesceWindowSeconds:
246 + schedule.type === "webhook" && schedule.dedupe?.coalesceWindowSeconds !== undefined
247 + ? String(schedule.dedupe.coalesceWindowSeconds)
248 + : "",
249 maxDeliveryAgeMinutes:
250 schedule.type === "webhook" && schedule.maxDeliveryAgeMinutes != null
251 ? String(schedule.maxDeliveryAgeMinutes)
docs/user/project-settings.md
+14
@@ -118,6 +118,20 @@ compare as text, and a value that is an object or a list only passes
118 **exists**. Separate **is one of** values with commas; a value can't contain
119 a comma or start or end with a space.
120
121 +To suppress repeated deliveries, set a **Replay key** template, such as
122 +`{{headers.delivery-id}}`. The same key is ignored for seven days per task.
123 +To combine a burst, set a **Coalesce key**, such as `{{body.id}}`, and a window
124 +of 10–3600 seconds (120 by default). One run uses the latest body, headers,
125 +and query when the window from the first delivery closes; later requests
126 +do not extend it. Pending windows survive a server restart. Keys must contain
127 +request placeholders; if any placeholder is missing, or the render is empty,
128 +that key's rule is skipped. Clear the templates to turn dedupe off.
129 +
130 +Duplicate and coalesced log entries share a limit of 10 per task per minute.
131 +Additional deliveries are accepted without a log entry, and coalesced deliveries
132 +always replace the persisted payload. Coalesced updates count toward a separate
133 +60-per-minute validation limit; exceeding it receives HTTP 429.
134 +
135 On desktop and web, pick **Deliveries** from a task's menu to see recent
136 requests and the prompt each one produced.
137
packages/contracts/src/scheduledTask.test.ts
+47 -4
@@ -2,14 +2,49 @@ import { describe, expect, it } from "vite-plus/test";
2 import * as Schema from "effect/Schema";
3
4 import {
5 + isWebhookDedupeTemplate,
6 MIN_SCHEDULED_TASK_INTERVAL_MS,
7 ScheduledTaskListWebhookDeliveriesResult,
8 ScheduledTaskSchedule,
9 ScheduledTaskUpsertSchedule,
10 + ScheduledTaskWebhookDeliverySummary,
11 } from "./scheduledTask.ts";
12
13 const decodeSchedule = Schema.decodeUnknownSync(ScheduledTaskSchedule);
14 const decodeUpsertSchedule = Schema.decodeUnknownSync(ScheduledTaskUpsertSchedule);
15 +const decodeDeliverySummary = Schema.decodeSync(ScheduledTaskWebhookDeliverySummary);
16 +const decodeDeliveryList = Schema.decodeUnknownSync(ScheduledTaskListWebhookDeliveriesResult);
17 +
18 +it("rejects a long unterminated dotted dedupe key without excessive backtracking", () => {
19 + const template = "{{body." + "a.".repeat(5000) + "x";
20 + const startedAt = performance.now();
21 + const valid = isWebhookDedupeTemplate(template);
22 + const elapsedMs = performance.now() - startedAt;
23 + expect(valid).toBe(false);
24 + expect(elapsedMs).toBeLessThan(50);
25 + for (const field of ["replayKey", "coalesceKey"]) {
26 + expect(() =>
27 + decodeUpsertSchedule({ type: "webhook", dedupe: { [field]: template } }),
28 + ).toThrow();
29 + }
30 +});
31 +
32 +it.each(["duplicate", "coalesced", "future_outcome"])("decodes delivery outcome %s", (outcome) => {
33 + expect(
34 + decodeDeliverySummary({
35 + id: "delivery:test",
36 + taskId: "scheduled-task:test",
37 + receivedAt: "2026-10-20T00:00:00.000Z",
38 + method: "POST",
39 + contentType: null,
40 + bodyBytes: 0,
41 + outcome,
42 + signatureVerified: false,
43 + missingFields: [],
44 + error: null,
45 + }).outcome,
46 + ).toBe(outcome);
47 +});
48
49 describe("ScheduledTaskSchedule", () => {
50 it("keeps legacy sub-minute persisted schedules readable", () => {
@@ -52,6 +87,15 @@ describe("ScheduledTaskUpsertSchedule", () => {
87 });
88 });
89
90 +it.each(["literal", " ", "\t\n", "{{body.bad", "{{unknown.field}}"])(
91 + "rejects invalid dedupe template %s",
92 + (replayKey) => {
93 + expect(() => decodeUpsertSchedule({ type: "webhook", dedupe: { replayKey } })).toThrow();
94 + expect(() =>
95 + decodeUpsertSchedule({ type: "webhook", dedupe: { coalesceKey: replayKey } }),
96 + ).toThrow();
97 + },
98 +);
99 describe("webhook filters", () => {
100 const accepts = (filters: unknown) => {
101 try {
@@ -114,9 +158,8 @@ describe("ScheduledTaskWebhookDeliverySummary", () => {
158 missingFields: [],
159 error: null,
160 };
117 - expect(
118 - Schema.decodeUnknownSync(ScheduledTaskListWebhookDeliveriesResult)({ deliveries: [summary] })
119 - .deliveries[0]?.outcome,
120 - ).toBe("some_future_outcome");
161 + expect(decodeDeliveryList({ deliveries: [summary] }).deliveries[0]?.outcome).toBe(
162 + "some_future_outcome",
163 + );
164 });
165 });
packages/contracts/src/scheduledTask.ts
+26
@@ -83,6 +83,26 @@ const WebhookMaxDeliveryAgeMinutes = Schema.Int.check(
83 "Skip requests the relay held longer than this many minutes while the environment was offline. Null runs every request.",
84 });
85
86 +export function isWebhookDedupeTemplate(template: string): boolean {
87 + const placeholders = /\{\{\s*(?:(?:body|headers|query)(?:\.[^{}.\s]+)*|request)\s*\}\}/g;
88 + const matches = [...template.matchAll(placeholders)];
89 + const remainder = template.replace(placeholders, "");
90 + return matches.length > 0 && !remainder.includes("{{") && !remainder.includes("}}");
91 +}
92 +
93 +const WebhookDedupeTemplate = Schema.String.check(Schema.makeFilter(isWebhookDedupeTemplate));
94 +
95 +const ScheduledTaskWebhookDedupe = Schema.Struct({
96 + replayKey: Schema.optional(WebhookDedupeTemplate),
97 + coalesceKey: Schema.optional(WebhookDedupeTemplate),
98 + coalesceWindowSeconds: Schema.optional(
99 + Schema.Int.check(Schema.isBetween({ minimum: 10, maximum: 3600 })),
100 + ),
101 +}).annotate({
102 + description:
103 + "Replay key template (7 days), coalesce key template, and fixed window in seconds (default 120).",
104 +});
105 +
106 export const MAX_WEBHOOK_FILTERS = 16;
107
108 const WebhookFilterPath = TrimmedNonEmptyString.check(
@@ -132,6 +152,7 @@ const ScheduledTaskWebhookSchedule = Schema.Struct({
152 }),
153 signature: Schema.NullOr(ScheduledTaskWebhookSignature),
154 maxDeliveryAgeMinutes: Schema.optional(Schema.NullOr(WebhookMaxDeliveryAgeMinutes)),
155 + dedupe: Schema.optional(ScheduledTaskWebhookDedupe),
156 filters: Schema.optional(ScheduledTaskWebhookFilters),
157 }).annotate({
158 description:
@@ -159,6 +180,7 @@ const ScheduledTaskUpsertWebhookSchedule = Schema.Struct({
180 description: "Signature check; omit or null to accept requests by URL token only.",
181 }),
182 maxDeliveryAgeMinutes: Schema.optional(Schema.NullOr(WebhookMaxDeliveryAgeMinutes)),
183 + dedupe: Schema.optional(ScheduledTaskWebhookDedupe),
184 filters: Schema.optional(ScheduledTaskWebhookFilters),
185 }).annotate({
186 description:
@@ -294,6 +316,8 @@ export type ScheduledTaskWebhookDeliveryId = typeof ScheduledTaskWebhookDelivery
316 export const ScheduledTaskWebhookDeliveryOutcome = Schema.Literals([
317 "accepted",
318 "dispatch_failed",
319 + "duplicate",
320 + "coalesced",
321 "rejected_signature",
322 "disabled",
323 "rate_limited",
@@ -312,6 +336,7 @@ export const ScheduledTaskWebhookDeliverySummary = Schema.Struct({
336 // Any string, so an outcome added later doesn't break a client's whole list.
337 outcome: Schema.Union([ScheduledTaskWebhookDeliveryOutcome, Schema.String]),
338 signatureVerified: Schema.Boolean,
339 + pendingWindow: Schema.optional(Schema.Boolean),
340 missingFields: Schema.Array(Schema.String),
341 error: Schema.NullOr(Schema.String),
342 });
@@ -348,6 +373,7 @@ export type ScheduledTaskListWebhookDeliveriesResult =
373 export const ScheduledTaskGetWebhookDeliveryInput = Schema.Struct({
374 id: ScheduledTaskId,
375 deliveryId: ScheduledTaskWebhookDeliveryId,
376 + acceptAnyOutcome: Schema.optional(Schema.Boolean),
377 });
378 export type ScheduledTaskGetWebhookDeliveryInput = typeof ScheduledTaskGetWebhookDeliveryInput.Type;
379