The best way to control Claude Code, Codex and more from any computer or phone
feat(scheduled-tasks): replay dedupe and coalescing for webhook deliveries (#22234)
Julius Marminge committed
Oct 20, 2026 at 13:49 UTC
2ebe280e5602f23fc2742297da3d3b6f2b1b8fca
20 files changed
+3123
-326
apps/mobile/src/features/settings/SettingsScheduledTasksRouteScreen.tsx
+42
@@ -716,6 +716,13 @@ function TaskForm({
716
Dialog.alert("Invalid filter", parsedFilters.error);
717
return;
718
}
719
+ if (draft.schedule.mode === "webhook" && schedule === null) {
720
+ Dialog.alert(
721
+ "Invalid webhook settings",
722
+ "Each dedupe key must contain a valid request placeholder. The coalesce window must be whole seconds from 10 to 3600, or blank.",
723
+ );
724
+ return;
725
+ }
726
if (
727
!draft.title.trim() ||
728
!draft.prompt.trim() ||
@@ -1060,6 +1067,41 @@ function TaskForm({
1067
setDraft({ ...draft, schedule: { ...draft.schedule, filters } })
1068
}
1069
/>
1070
+ <FormField
1071
+ label="Replay key template"
1072
+ value={draft.schedule.replayKey}
1073
+ placeholder="{{headers.delivery-id}}"
1074
+ disabled={saving}
1075
+ borderTop
1076
+ onChange={(replayKey) =>
1077
+ setDraft({ ...draft, schedule: { ...draft.schedule, replayKey } })
1078
+ }
1079
+ />
1080
+ <FormField
1081
+ label="Coalesce key template"
1082
+ value={draft.schedule.coalesceKey}
1083
+ placeholder="{{body.id}}"
1084
+ disabled={saving}
1085
+ borderTop
1086
+ onChange={(coalesceKey) =>
1087
+ setDraft({ ...draft, schedule: { ...draft.schedule, coalesceKey } })
1088
+ }
1089
+ />
1090
+ <FormField
1091
+ label="Coalesce window (seconds, 10–3600)"
1092
+ value={draft.schedule.coalesceWindowSeconds}
1093
+ placeholder="120"
1094
+ keyboardType="number-pad"
1095
+ disabled={saving}
1096
+ borderTop
1097
+ onChange={(coalesceWindowSeconds) =>
1098
+ setDraft({ ...draft, schedule: { ...draft.schedule, coalesceWindowSeconds } })
1099
+ }
1100
+ />
1101
+ <Text className="px-4 pb-3 text-sm text-foreground-muted">
1102
+ Replay keys suppress repeats for 7 days. Coalescing runs the latest payload when the
1103
+ window from the first delivery closes. Empty keys run normally.
1104
+ </Text>
1105
</>
1106
) : (
1107
<>
apps/mobile/src/features/settings/scheduledTaskDraft.test.ts
+43
-1
@@ -30,6 +30,37 @@ describe("scheduleDraftForTask", () => {
30
expect(scheduleFromDraft(scheduleDraftForTask({ schedule }))).toEqual(schedule);
31
});
32
33
+ it("round-trips dedupe settings and rejects invalid windows", () => {
34
+ const dedupe = {
35
+ replayKey: "{{headers.delivery-id}}",
36
+ coalesceKey: "{{body.id}}",
37
+ coalesceWindowSeconds: 3600,
38
+ };
39
+ const draft = scheduleDraftForTask({ schedule: { type: "webhook", signature: null, dedupe } });
40
+ expect(scheduleFromDraft(draft)).toMatchObject({ dedupe });
41
+ expect(
42
+ scheduleFromDraft({
43
+ ...draft,
44
+ replayKey: " \n ",
45
+ coalesceKey: " \t ",
46
+ coalesceWindowSeconds: "",
47
+ }),
48
+ ).toMatchObject({ dedupe: {} });
49
+ expect(
50
+ scheduleFromDraft({
51
+ ...draft,
52
+ replayKey: ` ${dedupe.replayKey} `,
53
+ coalesceKey: `\t${dedupe.coalesceKey}\n`,
54
+ }),
55
+ ).toMatchObject({ dedupe });
56
+ expect(
57
+ scheduleFromDraft({ ...draft, replayKey: "", coalesceKey: "", coalesceWindowSeconds: "" }),
58
+ ).toMatchObject({ dedupe: {} });
59
+ for (const coalesceWindowSeconds of ["9", "3601", "1.5", "abc"]) {
60
+ expect(scheduleFromDraft({ ...draft, coalesceWindowSeconds })).toBeNull();
61
+ }
62
+ });
63
+
64
it("round-trips a webhook schedule without a signature", () => {
65
const draft = scheduleDraftForTask({ schedule: { type: "webhook", signature: null } });
66
expect(draft.mode).toBe("webhook");
@@ -37,6 +68,7 @@ describe("scheduleDraftForTask", () => {
68
type: "webhook",
69
signature: null,
70
maxDeliveryAgeMinutes: null,
71
+ dedupe: {},
72
});
73
});
74
@@ -50,6 +82,7 @@ describe("scheduleDraftForTask", () => {
82
expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: "0" })).toBeNull();
83
expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: "" })).toMatchObject({
84
maxDeliveryAgeMinutes: null,
85
+ dedupe: {},
86
});
87
});
88
@@ -62,7 +95,7 @@ describe("scheduleDraftForTask", () => {
95
const saved = scheduleFromDraft(
96
scheduleDraftForTask({ schedule: { type: "webhook", signature } }),
97
);
65
- expect(saved).toEqual({ type: "webhook", signature, maxDeliveryAgeMinutes: null });
98
+ expect(saved).toEqual({ type: "webhook", signature, maxDeliveryAgeMinutes: null, dedupe: {} });
99
expect(saved?.type === "webhook" && saved.signature && "secret" in saved.signature).toBe(false);
100
});
101
@@ -369,3 +402,12 @@ describe("scheduled task model defaults", () => {
402
).toBeNull();
403
});
404
});
405
+
406
+it("rejects dedupe keys without valid request placeholders", () => {
407
+ expect(
408
+ scheduleFromDraft({ ...DEFAULT_SCHEDULE, mode: "webhook", replayKey: "literal" }),
409
+ ).toBeNull();
410
+ expect(
411
+ scheduleFromDraft({ ...DEFAULT_SCHEDULE, mode: "webhook", coalesceKey: "{{body.bad" }),
412
+ ).toBeNull();
413
+});
apps/mobile/src/features/settings/scheduledTaskDraft.ts
+38
@@ -1,3 +1,4 @@
1
+import { isWebhookDedupeTemplate } from "@t3tools/contracts";
2
import type {
3
ModelSelection,
4
ServerConfig,
@@ -56,6 +57,9 @@ export type ScheduleDraft = {
57
readonly intervalMinutes: string;
58
readonly signature: ScheduledTaskWebhookSignature | null;
59
readonly maxDeliveryAgeMinutes: string;
60
+ readonly replayKey: string;
61
+ readonly coalesceKey: string;
62
+ readonly coalesceWindowSeconds: string;
63
readonly filters: ReadonlyArray<WebhookFilterDraft>;
64
/** The filter rows the form opened with. */
65
readonly savedFilters: ReadonlyArray<WebhookFilterDraft>;
@@ -68,6 +72,9 @@ export const DEFAULT_SCHEDULE: ScheduleDraft = {
72
intervalMinutes: "15",
73
signature: null,
74
maxDeliveryAgeMinutes: "",
75
+ replayKey: "",
76
+ coalesceKey: "",
77
+ coalesceWindowSeconds: "",
78
filters: [],
79
savedFilters: [],
80
};
@@ -93,6 +100,12 @@ export function scheduleDraftForTask(task: Pick<ScheduledTask, "schedule">): Sch
100
...DEFAULT_SCHEDULE,
101
mode: "webhook",
102
signature: task.schedule.signature,
103
+ replayKey: task.schedule.dedupe?.replayKey ?? "",
104
+ coalesceKey: task.schedule.dedupe?.coalesceKey ?? "",
105
+ coalesceWindowSeconds:
106
+ task.schedule.dedupe?.coalesceWindowSeconds === undefined
107
+ ? ""
108
+ : String(task.schedule.dedupe.coalesceWindowSeconds),
109
maxDeliveryAgeMinutes:
110
task.schedule.maxDeliveryAgeMinutes == null
111
? ""
@@ -107,6 +120,19 @@ export function scheduleFromDraft(draft: ScheduleDraft): ScheduledTaskUpsertSche
120
if (draft.mode === "webhook") {
121
const maxDeliveryAgeMinutes = parseMaxDeliveryAge(draft.maxDeliveryAgeMinutes);
122
if (maxDeliveryAgeMinutes === undefined) return null;
123
+ const replayKey = draft.replayKey.trim();
124
+ const coalesceKey = draft.coalesceKey.trim();
125
+ if ([replayKey, coalesceKey].some((key) => key && !isWebhookDedupeTemplate(key))) return null;
126
+ const coalesceWindowSeconds = draft.coalesceWindowSeconds?.trim()
127
+ ? Number(draft.coalesceWindowSeconds)
128
+ : undefined;
129
+ if (
130
+ coalesceWindowSeconds !== undefined &&
131
+ (!Number.isInteger(coalesceWindowSeconds) ||
132
+ coalesceWindowSeconds < 10 ||
133
+ coalesceWindowSeconds > 3600)
134
+ )
135
+ return null;
136
const parsedFilters = parseWebhookFilterDrafts(draft.filters);
137
if (!("filters" in parsedFilters)) return null;
138
// No secret is sent, so the server keeps the stored one.
@@ -121,6 +147,15 @@ export function scheduleFromDraft(draft: ScheduleDraft): ScheduledTaskUpsertSche
147
prefix: draft.signature.prefix,
148
},
149
maxDeliveryAgeMinutes,
150
+ ...(replayKey || coalesceKey || coalesceWindowSeconds !== undefined
151
+ ? {
152
+ dedupe: {
153
+ ...(replayKey ? { replayKey } : {}),
154
+ ...(coalesceKey ? { coalesceKey } : {}),
155
+ ...(coalesceWindowSeconds === undefined ? {} : { coalesceWindowSeconds }),
156
+ },
157
+ }
158
+ : { dedupe: {} }),
159
// Only edited filters are sent; the server keeps the stored ones otherwise.
160
...(webhookFilterDraftsChanged(draft.savedFilters, draft.filters)
161
? { filters: parsedFilters.filters }
@@ -179,6 +214,9 @@ function draftSignature(draft: ScheduledTaskDraft): string {
214
[...draft.schedule.weekdays].sort((a, b) => a - b),
215
draft.schedule.intervalMinutes,
216
draft.schedule.maxDeliveryAgeMinutes,
217
+ draft.schedule.replayKey,
218
+ draft.schedule.coalesceKey,
219
+ draft.schedule.coalesceWindowSeconds,
220
draft.schedule.filters.map(({ path, op, value }) => [path, op, value]),
221
draft.workspace,
222
draft.baseRef,
apps/server/src/mcp/toolkits/orchestrator/tools.ts
+2
-2
@@ -116,7 +116,7 @@ const WorkflowStopTool = Tool.make("t3_workflow_stop", {
116
117
export const ScheduleTaskTool = Tool.make("schedule_task", {
118
description:
119
- "Create persistent work in the app scheduler that runs even when no turn is active. Pass schedule as a STRUCTURED OBJECT, never JSON text. Timers: {type:'interval', everyMs:3600000} is hourly; {type:'fixed_time', timeOfDay:'09:00', weekdays:[1,2,3,4,5]} is weekday mornings; report the returned nextRunAt. Webhooks: {type:'webhook'} runs once per request to a generated URL. The run sees the request ONLY through prompt placeholders: {{body.path}} (e.g. {{body.action}}, {{body.release.tag_name}}), {{headers.name}}, {{query.name}}, {{body}}, or {{request}} (method, headers with credentials redacted, and body). Each value arrives framed as untrusted request data, and $skill, @file or /command mentions inside it do not act; without a signature, anyone with the URL can trigger a run. For a sender that signs requests, first call request_secret so the user enters the secret privately (never ask for it in chat or invent one), then set signature with the returned secretRef, e.g. GitHub: {type:'webhook', signature:{header:'x-hub-signature-256', encoding:'hex', prefix:'sha256=', secretRef}}. To run only for some requests, add filters, all of which must match, e.g. filters:[{path:'headers.x-github-event', op:'equals', value:'pull_request'}, {path:'body.action', op:'in', value:['opened','reopened']}]; ops are equals, notEquals, in, startsWith and exists. The result's webhookUrl is the public URL to give the user; if it is absent, this environment has no T3 Connect managed tunnel, so tell the user to enable T3 Connect remote access rather than sharing a path. Omit projectId for this thread's project. In this thread's project, runs post into THIS thread by default (bindToCurrentThread=true), which suits an orchestrator that sees every trigger, delegates work, and can dedupe against what is in flight; use false only when the user wants a fresh top-level thread per run. Elsewhere each run launches a fresh thread. Provider, model, and runtime settings inherit from this thread, or from the project default when there is no calling thread.",
119
+ "Create persistent work in the app scheduler that runs even when no turn is active. Pass schedule as a STRUCTURED OBJECT, never JSON text. Timers: {type:'interval', everyMs:3600000} is hourly; {type:'fixed_time', timeOfDay:'09:00', weekdays:[1,2,3,4,5]} is weekday mornings; report the returned nextRunAt. Webhook dedupe accepts replayKey and coalesceKey templates with coalesceWindowSeconds (10–3600, default 120); replay keys expire after 7 days and coalescing keeps the latest payload in a window measured from its first delivery. Webhooks: {type:'webhook'} runs once per request to a generated URL. The run sees the request ONLY through prompt placeholders: {{body.path}} (e.g. {{body.action}}, {{body.release.tag_name}}), {{headers.name}}, {{query.name}}, {{body}}, or {{request}} (method, headers with credentials redacted, and body). Each value arrives framed as untrusted request data, and $skill, @file or /command mentions inside it do not act; without a signature, anyone with the URL can trigger a run. For a sender that signs requests, first call request_secret so the user enters the secret privately (never ask for it in chat or invent one), then set signature with the returned secretRef, e.g. GitHub: {type:'webhook', signature:{header:'x-hub-signature-256', encoding:'hex', prefix:'sha256=', secretRef}}. To run only for some requests, add filters, all of which must match, e.g. filters:[{path:'headers.x-github-event', op:'equals', value:'pull_request'}, {path:'body.action', op:'in', value:['opened','reopened']}]; ops are equals, notEquals, in, startsWith and exists. The result's webhookUrl is the public URL to give the user; if it is absent, this environment has no T3 Connect managed tunnel, so tell the user to enable T3 Connect remote access rather than sharing a path. Omit projectId for this thread's project. In this thread's project, runs post into THIS thread by default (bindToCurrentThread=true), which suits an orchestrator that sees every trigger, delegates work, and can dedupe against what is in flight; use false only when the user wants a fresh top-level thread per run. Elsewhere each run launches a fresh thread. Provider, model, and runtime settings inherit from this thread, or from the project default when there is no calling thread.",
120
parameters: OrchestratorMcpScheduleTaskInput,
121
success: OrchestratorMcpScheduleTaskResult,
122
failure: OrchestratorMcpFailure,
@@ -143,7 +143,7 @@ const ListScheduledTasksTool = Tool.make("list_scheduled_tasks", {
143
144
const UpdateScheduledTaskTool = Tool.make("update_scheduled_task", {
145
description:
146
- "Update an existing scheduled task by scheduledTaskId (from list_scheduled_tasks). Only the provided fields change; omit a field to leave it as-is. Use enabled=false to pause a task without deleting it. Set bindToCurrentThread to move the task between posting into this thread and launching a fresh thread per run. A webhook schedule without filters keeps the stored filters; pass filters to replace them, or filters:[] to clear them.",
146
+ "Update an existing scheduled task by scheduledTaskId (from list_scheduled_tasks). Webhook schedule dedupe accepts replayKey, coalesceKey, and coalesceWindowSeconds (10–3600, default 120) to suppress replays for 7 days and retain the latest payload in each fixed window. Only the provided fields change; omit a field to leave it as-is. Use enabled=false to pause a task without deleting it. Set bindToCurrentThread to move the task between posting into this thread and launching a fresh thread per run. A webhook schedule without filters keeps the stored filters; pass filters to replace them, or filters:[] to clear them.",
147
parameters: OrchestratorMcpUpdateScheduledTaskInput,
148
success: OrchestratorMcpScheduleTaskResult,
149
failure: OrchestratorMcpFailure,
apps/server/src/persistence/Migrations.ts
+2
@@ -69,6 +69,7 @@ import Migration0063 from "./Migrations/063_DropUnusedV1Projections.ts";
69
import Migration0064 from "./Migrations/064_CanonicalizeThreadPullRequestLinks.ts";
70
import Migration0065 from "./Migrations/065_CodexUsageNativeTurnIndex.ts";
71
import Migration0066 from "./Migrations/066_RepairClientSettlementTimestamps.ts";
72
+import Migration0067 from "./Migrations/067_WebhookDedupe.ts";
73
74
export const migrationEntries = [
75
[1, "OrchestrationEvents", Migration0001],
@@ -138,6 +139,7 @@ export const migrationEntries = [
139
[64, "CanonicalizeThreadPullRequestLinks", Migration0064],
140
[65, "CodexUsageNativeTurnIndex", Migration0065],
141
[66, "RepairClientSettlementTimestamps", Migration0066],
142
+ [67, "WebhookDedupe", Migration0067],
143
] as const;
144
145
export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const);
apps/server/src/persistence/Migrations/055_OrchestrationV2.test.ts
+3
-1
@@ -15,7 +15,7 @@ layer("055_OrchestrationV2", (it) => {
15
Effect.sync(() => {
16
assert.deepStrictEqual(
17
migrationEntries.map(([id]) => id),
18
- Array.from({ length: 66 }, (_, index) => index + 1),
18
+ Array.from({ length: 67 }, (_, index) => index + 1),
19
);
20
}),
21
);
@@ -40,6 +40,7 @@ layer("055_OrchestrationV2", (it) => {
40
[64, "CanonicalizeThreadPullRequestLinks"],
41
[65, "CodexUsageNativeTurnIndex"],
42
[66, "RepairClientSettlementTimestamps"],
43
+ [67, "WebhookDedupe"],
44
]);
45
assert.deepStrictEqual(yield* runMigrations(), []);
46
@@ -72,6 +73,7 @@ layer("055_OrchestrationV2", (it) => {
73
{ migration_id: 64, name: "CanonicalizeThreadPullRequestLinks" },
74
{ migration_id: 65, name: "CodexUsageNativeTurnIndex" },
75
{ migration_id: 66, name: "RepairClientSettlementTimestamps" },
76
+ { migration_id: 67, name: "WebhookDedupe" },
77
]);
78
79
const tables = yield* sql<{ readonly name: string }>`
apps/server/src/persistence/Migrations/067_WebhookDedupe.ts
new
+29
@@ -0,0 +1,29 @@
1
+import * as Effect from "effect/Effect";
2
+import * as SqlClient from "effect/sql/SqlClient";
3
+
4
+export default Effect.gen(function* () {
5
+ const sql = yield* SqlClient.SqlClient;
6
+ yield* sql`ALTER TABLE scheduled_tasks ADD COLUMN last_run_webhook_delivery_id TEXT`;
7
+ yield* sql`CREATE TABLE scheduled_task_webhook_replay_keys (
8
+ task_id TEXT NOT NULL REFERENCES scheduled_tasks(task_id) ON DELETE CASCADE,
9
+ replay_key TEXT NOT NULL,
10
+ delivery_id TEXT NOT NULL,
11
+ expires_at INTEGER NOT NULL,
12
+ PRIMARY KEY (task_id, replay_key)
13
+ )`;
14
+ yield* sql`CREATE INDEX idx_webhook_replay_expiry ON scheduled_task_webhook_replay_keys(expires_at)`;
15
+ yield* sql`CREATE INDEX idx_webhook_replay_delivery ON scheduled_task_webhook_replay_keys(delivery_id)`;
16
+ yield* sql`CREATE TABLE scheduled_task_webhook_pending (
17
+ delivery_id TEXT PRIMARY KEY,
18
+ task_id TEXT NOT NULL REFERENCES scheduled_tasks(task_id) ON DELETE CASCADE,
19
+ coalesce_key TEXT NOT NULL,
20
+ due_at INTEGER NOT NULL,
21
+ state TEXT NOT NULL DEFAULT 'pending',
22
+ task_json TEXT NOT NULL,
23
+ request_json TEXT NOT NULL,
24
+ delivery_ids_json TEXT NOT NULL
25
+ )`;
26
+ yield* sql`CREATE INDEX idx_webhook_pending_due ON scheduled_task_webhook_pending(state, due_at)`;
27
+ yield* sql`CREATE UNIQUE INDEX idx_webhook_pending_key
28
+ ON scheduled_task_webhook_pending(task_id, coalesce_key) WHERE state = 'pending'`;
29
+});
apps/server/src/persistence/reconcileV2PreviewMigration.test.ts
+2
@@ -46,6 +46,7 @@ describe("V2 preview upgrade", () => {
46
[64, "CanonicalizeThreadPullRequestLinks"],
47
[65, "CodexUsageNativeTurnIndex"],
48
[66, "RepairClientSettlementTimestamps"],
49
+ [67, "WebhookDedupe"],
50
]);
51
assert.deepStrictEqual(yield* runMigrations(), []);
52
assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports);
@@ -135,6 +136,7 @@ describe("V2 preview upgrade", () => {
136
[64, "CanonicalizeThreadPullRequestLinks"],
137
[65, "CodexUsageNativeTurnIndex"],
138
[66, "RepairClientSettlementTimestamps"],
139
+ [67, "WebhookDedupe"],
140
]);
141
}).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))),
142
);
apps/server/src/scheduledTasks/ScheduledTaskService.ts
+581
-161
@@ -29,6 +29,7 @@ import * as Crypto from "effect/Crypto";
29
import * as Data from "effect/Data";
30
import * as DateTime from "effect/DateTime";
31
import * as Effect from "effect/Effect";
32
+import * as Exit from "effect/Exit";
33
import * as Duration from "effect/Duration";
34
import * as Metric from "effect/Metric";
35
import * as Layer from "effect/Layer";
@@ -54,14 +55,30 @@ import {
55
renderWebhookPrompt,
56
type WebhookRequest,
57
} from "./webhookTemplate.ts";
58
+import {
59
+ acceptWebhookDedupe,
60
+ acceptImmediateWebhookDedupe,
61
+ releaseWebhookReplay,
62
+ decodePendingTask,
63
+ decodePendingRequest,
64
+ dedupeError,
65
+} from "./webhookDedupe.ts";
66
import { describeFailedWebhookFilter, findFailedWebhookFilter } from "./webhookFilter.ts";
67
import { constantTimeEquals, verifyWebhookSignature } from "./webhookVerification.ts";
68
69
export const WEBHOOK_ROUTE_PREFIX = "/api/hooks";
70
+const isScheduledTaskError = Schema.is(ScheduledTaskError);
71
+const PROMPT_TOO_LONG = "The filled-in prompt is too long.";
72
const WEBHOOK_DELIVERY_RETENTION = 50;
62
-/** Kept apart from the rest, so a stream of filtered requests can't evict real deliveries. */
73
+const WEBHOOK_DEDUPE_DELIVERY_RETENTION = 20;
74
const WEBHOOK_FILTERED_DELIVERY_RETENTION = 20;
64
-/** The outcomes clients decoded before outcomes became open-ended. */
75
+const WEBHOOK_DEDUPE_LOG_PER_MINUTE = 10;
76
+const WEBHOOK_FILTERED_LOG_PER_MINUTE = 10;
77
+const WEBHOOK_DELIVERY_LOG_BODY_LIMIT = 64 * 1024;
78
+const WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT = 64 * 1024;
79
+const WEBHOOK_MAX_QUEUED_PER_TASK = 20;
80
+/** Accepted deliveries per task per minute, enforced here as well as on the relay because the tunnel hostname is public too. */
81
+const WEBHOOK_RATE_LIMIT_PER_MINUTE = 60;
82
const LEGACY_DELIVERY_OUTCOMES: ReadonlySet<string> = new Set([
83
"accepted",
84
"dispatch_failed",
@@ -70,16 +87,6 @@ const LEGACY_DELIVERY_OUTCOMES: ReadonlySet<string> = new Set([
87
"rate_limited",
88
"expired",
89
]);
73
-/**
74
- * Filtered requests take no run slot, so this bounds the log writes they
75
- * cause instead; past it a filtered request is only counted in metrics.
76
- */
77
-const WEBHOOK_FILTERED_LOG_PER_MINUTE = 10;
78
-const WEBHOOK_DELIVERY_LOG_BODY_LIMIT = 64 * 1024;
79
-const WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT = 64 * 1024;
80
-const WEBHOOK_MAX_QUEUED_PER_TASK = 20;
81
-/** Accepted deliveries per task per minute, enforced here as well as on the relay because the tunnel hostname is public too. */
82
-const WEBHOOK_RATE_LIMIT_PER_MINUTE = 60;
90
91
interface WebhookOrigin {
92
readonly relayHookBaseUrl: string | null;
@@ -111,7 +118,7 @@ class WebhookDeliverySkipped extends Data.TaggedError("WebhookDeliverySkipped")<
118
}> {}
119
120
interface RateWindow {
114
- readonly accepted: ReadonlyArray<number>;
121
+ readonly accepted: ReadonlyArray<{ readonly at: number }>;
122
readonly rejectedLogged: boolean;
123
}
124
@@ -126,6 +133,7 @@ export interface WebhookTriggerRequest extends WebhookRequest {
133
export type WebhookDeliveryOutcome =
134
| "accepted"
135
| "duplicate"
136
+ | "coalesced"
137
| "prompt_too_long"
138
| "queue_full"
139
| "rate_limited"
@@ -140,7 +148,7 @@ export type WebhookTriggerResult =
148
| {
149
readonly _tag: "accepted";
150
readonly deliveryId: ScheduledTaskWebhookDeliveryId;
143
- readonly outcome: "accepted" | "duplicate" | "prompt_too_long" | "filtered";
151
+ readonly outcome: "accepted" | "duplicate" | "coalesced" | "prompt_too_long" | "filtered";
152
}
153
| { readonly _tag: "not_found" }
154
| { readonly _tag: "rejected_signature" }
@@ -211,6 +219,7 @@ interface WebhookDeliveryRow {
219
readonly signature_verified: number;
220
readonly missing_fields_json: string;
221
readonly rendered_prompt: string | null;
222
+ readonly pending_window: number;
223
readonly error: string | null;
224
}
225
@@ -405,6 +414,13 @@ export const layer = Layer.effect(
414
// finish late release their own count, never a recreated task's.
415
const webhookQueued = yield* Ref.make<ReadonlyMap<string, number>>(new Map());
416
const webhookRateWindows = yield* Ref.make<ReadonlyMap<ScheduledTaskId, RateWindow>>(new Map());
417
+ const webhookRejectedRateWindows = yield* Ref.make<ReadonlyMap<ScheduledTaskId, RateWindow>>(
418
+ new Map(),
419
+ );
420
+ const webhookDedupeLogWindows = yield* Ref.make<
421
+ ReadonlyMap<ScheduledTaskId, ReadonlyArray<number>>
422
+ >(new Map());
423
+ const pendingInFlight = yield* Ref.make<ReadonlySet<string>>(new Set());
424
const webhookFilteredLogWindows = yield* Ref.make<
425
ReadonlyMap<ScheduledTaskId, ReadonlyArray<number>>
426
>(new Map());
@@ -627,11 +643,12 @@ export const layer = Layer.effect(
643
),
644
);
645
630
- const markRunning = (id: ScheduledTaskId, startedAtIso: string) =>
646
+ const markRunning = (id: ScheduledTaskId, startedAtIso: string, webhookDeliveryId?: string) =>
647
sql<{ task_id: string }>`
648
UPDATE scheduled_tasks
649
SET updated_at = ${startedAtIso},
650
last_run_at = ${startedAtIso},
651
+ last_run_webhook_delivery_id = ${webhookDeliveryId ?? null},
652
last_run_status = 'running',
653
last_run_error = NULL
654
WHERE task_id = ${id}
@@ -673,9 +690,10 @@ export const layer = Layer.effect(
690
// markCompleted, write a full terminal record so runDueTasks neither skips
691
// the task forever (it filters out 'running' rows) nor re-fires it
692
// immediately: the dispatch may already have gone out, so next_run_at must
676
- // advance and run_count must count the attempt.
693
+ // advance. Pending webhook attempts count only with their completion state;
694
+ // other attempts count here.
695
const releaseStuckRun = Effect.fn("ScheduledTaskService.releaseStuckRun")(
678
- function* (task: ScheduledTask, message: string) {
696
+ function* (task: ScheduledTask, message: string, pendingDeliveryId?: string) {
697
const now = yield* localNow;
698
const reread = yield* Effect.result(findTask(task.id));
699
if (Result.isSuccess(reread) && reread.success === null) return;
@@ -686,12 +704,15 @@ export const layer = Layer.effect(
704
last_run_error = ${message},
705
next_run_at = ${nextRunAt(source, now)},
706
updated_at = ${iso(now)},
689
- run_count = run_count + 1
707
+ run_count = run_count + CASE WHEN EXISTS (
708
+ SELECT 1 FROM scheduled_task_webhook_pending
709
+ WHERE delivery_id = ${pendingDeliveryId ?? null} AND state = 'running'
710
+ ) THEN 0 ELSE 1 END
711
WHERE task_id = ${task.id} AND last_run_status = 'running'
712
`;
713
yield* notifyChanged;
714
},
694
- (effect, task, _message) =>
715
+ (effect, task, _message, _pendingDeliveryId) =>
716
effect.pipe(
717
Effect.catch((cause) =>
718
Effect.logWarning("Could not release stuck schedule task run", {
@@ -705,7 +726,11 @@ export const layer = Layer.effect(
726
const runTask = Effect.fn("ScheduledTaskService.runTask")(function* (
727
task: ScheduledTask,
728
trigger: "scheduled" | "manual" | "webhook",
708
- webhook?: { readonly deliveryId: string; readonly prompt: string },
729
+ webhook?: {
730
+ readonly deliveryId: string;
731
+ readonly prompt: string;
732
+ readonly dispatched: Ref.Ref<boolean>;
733
+ },
734
) {
735
const reserved = yield* Ref.modify(activeRuns, (active) => {
736
if (active.has(task.id)) return [false, active] as const;
@@ -739,6 +764,12 @@ export const layer = Layer.effect(
764
}
765
return { task, threadId: null };
766
}
767
+ if (
768
+ webhook !== undefined &&
769
+ (yield* sql`SELECT 1 FROM scheduled_task_webhook_pending WHERE delivery_id = ${webhook.deliveryId} AND state = 'completed'`)
770
+ .length > 0
771
+ )
772
+ return { task: { ...active, lastRunStatus: "succeeded" as const }, threadId: null };
773
const parsedNextRunAt =
774
active.nextRunAt === null ? Option.none() : DateTime.make(active.nextRunAt);
775
if (
@@ -764,7 +795,22 @@ export const layer = Layer.effect(
795
if (reason !== null) return yield* new WebhookDeliverySkipped({ reason });
796
}
797
767
- yield* markRunning(active.id, startedAtIso);
798
+ if (webhook === undefined) yield* markRunning(active.id, startedAtIso);
799
+ else
800
+ yield* sql
801
+ .withTransaction(
802
+ Effect.gen(function* () {
803
+ yield* markRunning(active.id, startedAtIso, webhook.deliveryId);
804
+ // Recovery must distinguish this delivery from other windows waiting on the same task.
805
+ yield* sql`UPDATE scheduled_task_webhook_pending SET state = 'running' WHERE delivery_id = ${webhook.deliveryId}`;
806
+ }),
807
+ )
808
+ .pipe(
809
+ Effect.mapError((cause) =>
810
+ taskError("Could not start webhook run.", { taskId: task.id, cause }),
811
+ ),
812
+ );
813
+
814
yield* notifyChanged;
815
816
const fireKey =
@@ -821,8 +867,9 @@ export const layer = Layer.effect(
867
.pipe(Effect.as(ThreadId.make(active.threadId))),
868
);
869
824
- const completedAt = yield* localNow;
870
const runSucceeded = result._tag === "Success";
871
+ if (webhook !== undefined && runSucceeded) yield* Ref.set(webhook.dispatched, true);
872
+ const completedAt = yield* localNow;
873
const lastRunStatus = runSucceeded ? ("succeeded" as const) : ("failed" as const);
874
const lastRunError = runSucceeded ? null : errorMessage(result.cause);
875
const threadId = runSucceeded ? result.value : null;
@@ -841,7 +888,7 @@ export const layer = Layer.effect(
888
// startedAtIso in the guard ensures this writes only to the row this
889
// run marked as running — a task deleted mid-run and recreated with
890
// the same id (idempotent commandId replay) must not be stamped.
844
- yield* markCompleted({
891
+ const completion = markCompleted({
892
id: task.id,
893
completedAtIso: completed.updatedAt,
894
nextRunAtIso: completed.nextRunAt,
@@ -849,11 +896,32 @@ export const layer = Layer.effect(
896
error: lastRunError,
897
startedAtIso,
898
});
899
+ if (webhook === undefined) yield* completion;
900
+ else
901
+ yield* sql
902
+ .withTransaction(
903
+ Effect.gen(function* () {
904
+ const pending =
905
+ yield* sql`SELECT 1 FROM scheduled_task_webhook_pending WHERE delivery_id = ${webhook.deliveryId}`;
906
+ if (pending.length === 0) yield* completion;
907
+ else {
908
+ const counted =
909
+ yield* sql`UPDATE scheduled_task_webhook_pending SET state = ${runSucceeded ? "completed" : "failed"}
910
+ WHERE delivery_id = ${webhook.deliveryId} AND state NOT IN ('completed', 'failed') RETURNING delivery_id`;
911
+ if (counted.length > 0) yield* completion;
912
+ }
913
+ }),
914
+ )
915
+ .pipe(
916
+ Effect.mapError((cause) =>
917
+ taskError("Could not record webhook completion.", { taskId: task.id, cause }),
918
+ ),
919
+ );
920
yield* notifyChanged;
921
}
922
return { task: completed, threadId };
923
}).pipe(
856
- Effect.onError((cause) => releaseStuckRun(task, errorMessage(cause))),
924
+ Effect.onError((cause) => releaseStuckRun(task, errorMessage(cause), webhook?.deliveryId)),
925
Effect.ensuring(
926
Ref.update(activeRuns, (active) => {
927
const next = new Set(active);
@@ -930,7 +998,11 @@ export const layer = Layer.effect(
998
last_run_error = 'Run was interrupted by a server restart.',
999
next_run_at = ${nextRunAt(decoded.success, now)},
1000
updated_at = ${iso(now)},
933
- run_count = run_count + 1
1001
+ run_count = run_count + CASE WHEN EXISTS (
1002
+ SELECT 1 FROM scheduled_task_webhook_pending
1003
+ WHERE task_id IS ${row.task_id} AND state = 'running'
1004
+ AND delivery_id = scheduled_tasks.last_run_webhook_delivery_id
1005
+ ) THEN 0 ELSE 1 END
1006
WHERE task_id IS ${row.task_id} AND last_run_status = 'running'
1007
`;
1008
return;
@@ -944,7 +1016,11 @@ export const layer = Layer.effect(
1016
SET last_run_status = 'failed',
1017
last_run_error = 'Run was interrupted by a server restart.',
1018
updated_at = ${iso(now)},
947
- run_count = run_count + 1
1019
+ run_count = run_count + CASE WHEN EXISTS (
1020
+ SELECT 1 FROM scheduled_task_webhook_pending
1021
+ WHERE task_id IS ${row.task_id} AND state = 'running'
1022
+ AND delivery_id = scheduled_tasks.last_run_webhook_delivery_id
1023
+ ) THEN 0 ELSE 1 END
1024
WHERE task_id IS ${row.task_id} AND last_run_status = 'running'
1025
`;
1026
}),
@@ -1017,6 +1093,12 @@ export const layer = Layer.effect(
1093
prefix: input.schedule.signature.prefix,
1094
},
1095
maxDeliveryAgeMinutes: input.schedule.maxDeliveryAgeMinutes ?? null,
1096
+ ...(input.schedule.dedupe === undefined
1097
+ ? existingTask?.schedule.type === "webhook" &&
1098
+ existingTask.schedule.dedupe !== undefined
1099
+ ? { dedupe: existingTask.schedule.dedupe }
1100
+ : {}
1101
+ : { dedupe: input.schedule.dedupe }),
1102
...webhookFilters(
1103
// Omitted keeps the stored filters, so a client that predates
1104
// them can re-save a task without clearing them.
@@ -1139,6 +1221,16 @@ export const layer = Layer.effect(
1221
next.delete(id);
1222
return next;
1223
}),
1224
+ Ref.update(webhookRejectedRateWindows, (windows) => {
1225
+ const next = new Map(windows);
1226
+ next.delete(id);
1227
+ return next;
1228
+ }),
1229
+ Ref.update(webhookDedupeLogWindows, (windows) => {
1230
+ const next = new Map(windows);
1231
+ next.delete(id);
1232
+ return next;
1233
+ }),
1234
Ref.update(webhookFilteredLogWindows, (windows) => {
1235
const next = new Map(windows);
1236
next.delete(id);
@@ -1243,6 +1335,7 @@ export const layer = Layer.effect(
1335
bodyBytes: row.body_bytes,
1336
outcome: row.outcome as ScheduledTaskWebhookDeliveryOutcome,
1337
signatureVerified: row.signature_verified === 1,
1338
+ pendingWindow: row.outcome === "accepted" && row.pending_window === 1,
1339
missingFields: Option.getOrElse(decodeMissingFieldsJson(row.missing_fields_json), () => []),
1340
error: row.error,
1341
});
@@ -1251,15 +1344,17 @@ export const layer = Layer.effect(
1344
input,
1345
) =>
1346
sql<WebhookDeliveryRow>`
1254
- SELECT * FROM scheduled_task_webhook_deliveries
1255
- WHERE task_id = ${input.id}
1347
+ SELECT d.*, EXISTS (SELECT 1 FROM scheduled_task_webhook_pending p
1348
+ WHERE p.delivery_id = d.delivery_id AND p.state IN ('pending', 'dispatching')) AS pending_window
1349
+ FROM scheduled_task_webhook_deliveries d WHERE task_id = ${input.id}
1350
ORDER BY received_at DESC, rowid DESC
1351
`.pipe(
1352
Effect.map((rows) => ({
1259
- deliveries: (input.acceptAnyOutcome === true
1260
- ? rows
1261
- : rows.filter((row) => LEGACY_DELIVERY_OUTCOMES.has(row.outcome))
1262
- ).map(decodeDeliverySummary),
1353
+ deliveries: rows
1354
+ .filter(
1355
+ (row) => input.acceptAnyOutcome === true || LEGACY_DELIVERY_OUTCOMES.has(row.outcome),
1356
+ )
1357
+ .map(decodeDeliverySummary),
1358
})),
1359
Effect.mapError((cause) =>
1360
taskError("Could not list webhook deliveries.", { taskId: input.id, cause }),
@@ -1270,15 +1365,19 @@ export const layer = Layer.effect(
1365
"ScheduledTaskService.getWebhookDelivery",
1366
)(function* (input) {
1367
const rows = yield* sql<WebhookDeliveryRow>`
1273
- SELECT * FROM scheduled_task_webhook_deliveries
1274
- WHERE task_id = ${input.id} AND delivery_id = ${input.deliveryId}
1368
+ SELECT d.*, EXISTS (SELECT 1 FROM scheduled_task_webhook_pending p
1369
+ WHERE p.delivery_id = d.delivery_id AND p.state IN ('pending', 'dispatching')) AS pending_window
1370
+ FROM scheduled_task_webhook_deliveries d WHERE task_id = ${input.id} AND delivery_id = ${input.deliveryId}
1371
`.pipe(
1372
Effect.mapError((cause) =>
1373
taskError("Could not load webhook delivery.", { taskId: input.id, cause }),
1374
),
1375
);
1376
const row = rows[0];
1281
- if (row === undefined) {
1377
+ if (
1378
+ row === undefined ||
1379
+ (input.acceptAnyOutcome !== true && !LEGACY_DELIVERY_OUTCOMES.has(row.outcome))
1380
+ ) {
1381
return yield* taskError("Webhook delivery not found.", { taskId: input.id });
1382
}
1383
return {
@@ -1293,6 +1392,24 @@ export const layer = Layer.effect(
1392
};
1393
});
1394
1395
+ const deliveryRequestBody = (request: WebhookTriggerRequest) => {
1396
+ const truncated = request.body.byteLength > WEBHOOK_DELIVERY_LOG_BODY_LIMIT;
1397
+ return {
1398
+ truncated,
1399
+ body: truncated
1400
+ ? new TextDecoder().decode(request.body.subarray(0, WEBHOOK_DELIVERY_LOG_BODY_LIMIT))
1401
+ : request.bodyText,
1402
+ };
1403
+ };
1404
+ const replaceDeliveryRequest = (deliveryId: string, request: WebhookTriggerRequest) => {
1405
+ const logged = deliveryRequestBody(request);
1406
+ return sql`UPDATE scheduled_task_webhook_deliveries SET
1407
+ method = ${request.method}, query = ${redactQuery(request.query)},
1408
+ headers_json = ${encodeHeadersJson(redactHeaders(request.headers))}, body = ${logged.body},
1409
+ body_bytes = ${request.body.byteLength}, body_truncated = ${logged.truncated ? 1 : 0}
1410
+ WHERE delivery_id = ${deliveryId}`.pipe(Effect.asVoid, Effect.mapError(dedupeError));
1411
+ };
1412
+
1413
const recordDelivery = (input: {
1414
readonly id: string;
1415
readonly taskId: ScheduledTaskId;
@@ -1304,10 +1421,7 @@ export const layer = Layer.effect(
1421
readonly renderedPrompt: string | null;
1422
readonly error?: string;
1423
}) => {
1307
- const truncated = input.request.body.byteLength > WEBHOOK_DELIVERY_LOG_BODY_LIMIT;
1308
- const loggedBody = truncated
1309
- ? new TextDecoder().decode(input.request.body.subarray(0, WEBHOOK_DELIVERY_LOG_BODY_LIMIT))
1310
- : input.request.bodyText;
1424
+ const { truncated, body: loggedBody } = deliveryRequestBody(input.request);
1425
return sql
1426
.withTransaction(
1427
Effect.gen(function* () {
@@ -1338,17 +1452,28 @@ export const layer = Layer.effect(
1452
rendered_prompt = excluded.rendered_prompt,
1453
error = excluded.error
1454
`;
1341
- const filtered = input.outcome === "filtered" ? 1 : 0;
1455
+ // Each suppression bucket retains its own rows, never real run history.
1456
+ const bucket = (outcome: string, error: string | undefined) =>
1457
+ outcome === "filtered"
1458
+ ? "filtered"
1459
+ : outcome === "duplicate" ||
1460
+ outcome === "coalesced" ||
1461
+ outcome === "rate_limited" ||
1462
+ error === PROMPT_TOO_LONG
1463
+ ? "dedupe"
1464
+ : "main";
1465
+ const retentionBucket = bucket(input.outcome, input.error);
1466
+ const rowBucket = sql.literal(`CASE WHEN outcome = 'filtered' THEN 'filtered'
1467
+ WHEN outcome IN ('duplicate', 'coalesced', 'rate_limited') OR error = 'The filled-in prompt is too long.' THEN 'dedupe'
1468
+ ELSE 'main' END`);
1469
yield* sql`
1470
DELETE FROM scheduled_task_webhook_deliveries
1344
- WHERE task_id = ${input.taskId}
1345
- AND (outcome = 'filtered') = ${filtered}
1471
+ WHERE task_id = ${input.taskId} AND ${rowBucket} = ${retentionBucket}
1472
AND delivery_id NOT IN (
1473
SELECT delivery_id FROM scheduled_task_webhook_deliveries
1348
- WHERE task_id = ${input.taskId} AND (outcome = 'filtered') = ${filtered}
1349
- -- rowid breaks timestamp ties in arrival order; delivery ids are random.
1474
+ WHERE task_id = ${input.taskId} AND ${rowBucket} = ${retentionBucket}
1475
ORDER BY received_at DESC, rowid DESC
1351
- LIMIT ${filtered ? WEBHOOK_FILTERED_DELIVERY_RETENTION : WEBHOOK_DELIVERY_RETENTION}
1476
+ LIMIT ${retentionBucket === "main" ? WEBHOOK_DELIVERY_RETENTION : retentionBucket === "filtered" ? WEBHOOK_FILTERED_DELIVERY_RETENTION : WEBHOOK_DEDUPE_DELIVERY_RETENTION}
1477
)
1478
`;
1479
}),
@@ -1367,13 +1492,15 @@ export const layer = Layer.effect(
1492
WHERE delivery_id = ${deliveryId}
1493
`.pipe(Effect.ignore);
1494
1370
- /**
1371
- * Sliding one-minute window counting every request with a valid token,
1372
- * including ones the signature check later rejects.
1373
- */
1374
- const takeRateSlot = (id: ScheduledTaskId, nowMs: number) =>
1495
+ /** Sliding one-minute window for deliveries that can start work. */
1496
+ const takeRateSlot = (
1497
+ id: ScheduledTaskId,
1498
+ nowMs: number,
1499
+ rateWindows = webhookRateWindows,
1500
+ reservation = { at: nowMs },
1501
+ ) =>
1502
Ref.modify(
1376
- webhookRateWindows,
1503
+ rateWindows,
1504
(
1505
windows,
1506
): readonly [
@@ -1381,7 +1508,7 @@ export const layer = Layer.effect(
1508
ReadonlyMap<ScheduledTaskId, RateWindow>,
1509
] => {
1510
const current = windows.get(id);
1384
- const recent = (current?.accepted ?? []).filter((at) => nowMs - at < 60_000);
1511
+ const recent = (current?.accepted ?? []).filter((entry) => nowMs - entry.at < 60_000);
1512
if (recent.length >= WEBHOOK_RATE_LIMIT_PER_MINUTE) {
1513
const first = current?.rejectedLogged !== true;
1514
return [
@@ -1391,11 +1518,23 @@ export const layer = Layer.effect(
1518
}
1519
return [
1520
"allowed",
1394
- new Map(windows).set(id, { accepted: [...recent, nowMs], rejectedLogged: false }),
1521
+ new Map(windows).set(id, { accepted: [...recent, reservation], rejectedLogged: false }),
1522
];
1523
},
1524
);
1525
1526
+ const takeDedupeLogSlot = (id: ScheduledTaskId, nowMs: number) =>
1527
+ Ref.modify(webhookDedupeLogWindows, (windows) => {
1528
+ const recent = (windows.get(id) ?? []).filter((at) => nowMs - at < 60_000);
1529
+ return recent.length >= WEBHOOK_DEDUPE_LOG_PER_MINUTE
1530
+ ? ([false, windows] as const)
1531
+ : ([true, new Map(windows).set(id, [...recent, nowMs])] as const);
1532
+ });
1533
+ const releaseReplay = (deliveryId: string) =>
1534
+ releaseWebhookReplay(deliveryId).pipe(
1535
+ Effect.provideService(SqlClient.SqlClient, sql),
1536
+ Effect.mapError(dedupeError),
1537
+ );
1538
/** Sliding one-minute window of filtered requests written to the delivery log. */
1539
const takeFilteredLogSlot = (id: ScheduledTaskId, nowMs: number) =>
1540
Ref.modify(
@@ -1424,6 +1563,83 @@ export const layer = Layer.effect(
1563
1564
const serviceScope = yield* Effect.scope;
1565
1566
+ const releaseFailedDelivery = Effect.fn("ScheduledTaskService.releaseFailedDelivery")(
1567
+ function* (deliveryId: string) {
1568
+ yield* releaseReplay(deliveryId);
1569
+ yield* sql`DELETE FROM scheduled_task_webhook_relay_deliveries WHERE relay_delivery_id IN (
1570
+ SELECT substr(value, ${"delivery:relay:".length + 1}) FROM scheduled_task_webhook_pending, json_each(delivery_ids_json)
1571
+ WHERE delivery_id = ${deliveryId} AND value LIKE 'delivery:relay:%'
1572
+ )`.pipe(Effect.mapError(dedupeError));
1573
+ if (deliveryId.startsWith("delivery:relay:"))
1574
+ yield* sql`DELETE FROM scheduled_task_webhook_relay_deliveries WHERE relay_delivery_id = ${deliveryId.slice("delivery:relay:".length)}`.pipe(
1575
+ Effect.mapError(dedupeError),
1576
+ );
1577
+ },
1578
+ (effect) => sql.withTransaction(effect).pipe(Effect.mapError(dedupeError)),
1579
+ );
1580
+
1581
+ const runWebhookDelivery = Effect.fn("ScheduledTaskService.runWebhookDelivery")(function* (
1582
+ task: ScheduledTask,
1583
+ deliveryId: string,
1584
+ prompt: string,
1585
+ ) {
1586
+ const runOutcome = (outcome: "started" | "skipped" | "failed") =>
1587
+ Effect.all([
1588
+ Effect.annotateCurrentSpan({ "scheduled_task.webhook.run_outcome": outcome }),
1589
+ Metrics.increment(Metrics.webhookRunsTotal, { outcome }),
1590
+ ]);
1591
+ const dispatched = yield* Ref.make(false);
1592
+ return yield* runTask(task, "webhook", { deliveryId, prompt, dispatched }).pipe(
1593
+ Effect.flatMap(({ task: completed }) =>
1594
+ completed.lastRunStatus === "failed"
1595
+ ? runOutcome("failed").pipe(
1596
+ Effect.andThen(markDeliveryFailed(deliveryId, "The run failed to start.")),
1597
+ Effect.andThen(releaseFailedDelivery(deliveryId)),
1598
+ )
1599
+ : runOutcome("started"),
1600
+ ),
1601
+ Effect.catchTags({
1602
+ WebhookDeliverySkipped: (skipped) =>
1603
+ runOutcome("skipped").pipe(
1604
+ Effect.andThen(markDeliveryFailed(deliveryId, skipped.reason)),
1605
+ Effect.andThen(releaseFailedDelivery(deliveryId)),
1606
+ ),
1607
+ }),
1608
+ Effect.as(true),
1609
+ // Dispatch success owns the replay key even if later bookkeeping fails.
1610
+ Effect.catchCause((cause) =>
1611
+ Effect.gen(function* () {
1612
+ const started = yield* Ref.get(dispatched);
1613
+ yield* Effect.logWarning("Webhook dispatch failed", { taskId: task.id, cause });
1614
+ yield* runOutcome(started ? "started" : "failed");
1615
+ yield* markDeliveryFailed(
1616
+ deliveryId,
1617
+ started
1618
+ ? "The run started, but completion could not be recorded."
1619
+ : "The run failed to start.",
1620
+ );
1621
+ if (!started) {
1622
+ yield* sql
1623
+ .withTransaction(
1624
+ Effect.gen(function* () {
1625
+ const counted =
1626
+ yield* sql`UPDATE scheduled_task_webhook_pending SET state = 'failed'
1627
+ WHERE delivery_id = ${deliveryId} AND state = 'running' RETURNING delivery_id`;
1628
+ if (counted.length > 0)
1629
+ yield* sql`UPDATE scheduled_tasks SET run_count = run_count + 1
1630
+ WHERE task_id = ${task.id} AND created_at = ${task.createdAt}`;
1631
+ }),
1632
+ )
1633
+ .pipe(Effect.mapError(dedupeError));
1634
+ yield* releaseFailedDelivery(deliveryId);
1635
+ }
1636
+ // Pending windows retry the same command id to finish bookkeeping.
1637
+ return !started;
1638
+ }),
1639
+ ),
1640
+ );
1641
+ });
1642
+
1643
// Delivery attributes belong to the enclosing triggerWebhook span.
1644
const observeDelivery = Effect.fnUntraced(function* (
1645
request: WebhookTriggerRequest,
@@ -1571,71 +1787,168 @@ export const layer = Layer.effect(
1787
...(details.error === undefined ? {} : { error: details.error }),
1788
});
1789
1790
+ // Reject floods before signature work or rejection-log writes; valid
1791
+ // deliveries give this reservation back once validation succeeds.
1792
+ const rejectionReservation = { at: DateTime.toEpochMillis(now) };
1793
+ const rejectionSlot = yield* takeRateSlot(
1794
+ task.id,
1795
+ DateTime.toEpochMillis(now),
1796
+ webhookRejectedRateWindows,
1797
+ rejectionReservation,
1798
+ );
1799
+ if (rejectionSlot !== "allowed") {
1800
+ if (rejectionSlot === "first_rejected") yield* log("rate_limited");
1801
+ yield* observe("rate_limited");
1802
+ return yield* releaseClaim({ _tag: "rate_limited", outcome: "rate_limited" } as const);
1803
+ }
1804
+ if (!task.enabled) {
1805
+ yield* log("disabled");
1806
+ yield* observe("disabled");
1807
+ return { _tag: "disabled" as const };
1808
+ }
1809
const signature = schedule.signature;
1575
- const signatureVerified =
1576
- signature === null ||
1577
- (row.webhook_secret !== null &&
1810
+ if (signature !== null) {
1811
+ const verified =
1812
+ row.webhook_secret !== null &&
1813
verifyWebhookSignature({
1814
signature,
1815
secret: row.webhook_secret,
1816
headers: request.headers,
1817
body: request.body,
1583
- }));
1818
+ });
1819
+ if (!verified) {
1820
+ yield* log("rejected_signature");
1821
+ yield* observe("rejected_signature");
1822
+ return { _tag: "rejected_signature" as const };
1823
+ }
1824
+ }
1825
const maxAgeMinutes = schedule.maxDeliveryAgeMinutes ?? null;
1585
- const tooOld =
1826
+ if (
1827
maxAgeMinutes !== null &&
1828
DateTime.toEpochMillis(now) - DateTime.toEpochMillis(receivedAt) >
1588
- maxAgeMinutes * 60_000;
1589
-
1590
- // Filters: checked after the signature and age, and before the rate
1591
- // limit, so requests the task ignores never take a run's slot. The
1592
- // request is accepted so the sender doesn't retry it, and its relay
1593
- // claim is released because nothing ran.
1594
- if (task.enabled && signatureVerified && !tooOld) {
1595
- const failedFilter = findFailedWebhookFilter(schedule.filters ?? [], request);
1596
- if (failedFilter !== null) {
1597
- if (yield* takeFilteredLogSlot(task.id, DateTime.toEpochMillis(now))) {
1598
- yield* log("filtered", {
1599
- signatureVerified: signature !== null,
1600
- error: describeFailedWebhookFilter(failedFilter),
1601
- });
1602
- }
1603
- yield* observe("filtered");
1604
- return yield* releaseClaim({
1605
- _tag: "accepted" as const,
1606
- deliveryId,
1607
- outcome: "filtered" as const,
1608
- });
1609
- }
1829
+ maxAgeMinutes * 60_000
1830
+ ) {
1831
+ yield* log("expired", { signatureVerified: signature !== null });
1832
+ yield* observe("expired");
1833
+ return { _tag: "expired" as const };
1834
}
1835
1612
- // Only the first rejected request in a window is logged, so a flood
1613
- // cannot write rows or push the real deliveries out of the log.
1614
- const slot = yield* takeRateSlot(task.id, DateTime.toEpochMillis(now));
1615
- if (slot !== "allowed") {
1616
- if (slot === "first_rejected") yield* log("rate_limited");
1617
- yield* observe("rate_limited");
1836
+ const refundRejectionSlot = Ref.update(webhookRejectedRateWindows, (windows) => {
1837
+ const current = windows.get(task.id);
1838
+ if (current === undefined) return windows;
1839
+ return new Map(windows).set(task.id, {
1840
+ ...current,
1841
+ accepted: current.accepted.filter((entry) => entry !== rejectionReservation),
1842
+ });
1843
+ });
1844
+
1845
+ const failedFilter = findFailedWebhookFilter(schedule.filters ?? [], request);
1846
+ if (failedFilter !== null) {
1847
+ if (yield* takeFilteredLogSlot(task.id, DateTime.toEpochMillis(now))) {
1848
+ yield* log("filtered", {
1849
+ signatureVerified: signature !== null,
1850
+ error: describeFailedWebhookFilter(failedFilter),
1851
+ });
1852
+ }
1853
+ yield* refundRejectionSlot;
1854
+ yield* observe("filtered");
1855
return yield* releaseClaim({
1619
- _tag: "rate_limited" as const,
1620
- outcome: "rate_limited" as const,
1856
+ _tag: "accepted" as const,
1857
+ deliveryId,
1858
+ outcome: "filtered" as const,
1859
});
1860
}
1623
- if (!task.enabled) {
1624
- yield* log("disabled");
1625
- yield* observe("disabled");
1626
- return { _tag: "disabled" as const };
1861
+
1862
+ const runWindow = (yield* Ref.get(webhookRateWindows)).get(task.id);
1863
+ if (
1864
+ schedule.dedupe === undefined &&
1865
+ (runWindow?.accepted.filter((entry) => DateTime.toEpochMillis(now) - entry.at < 60_000)
1866
+ .length ?? 0) >= WEBHOOK_RATE_LIMIT_PER_MINUTE
1867
+ ) {
1868
+ if ((yield* takeRateSlot(task.id, DateTime.toEpochMillis(now))) === "first_rejected")
1869
+ yield* log("rate_limited");
1870
+ yield* observe("rate_limited");
1871
+ return yield* releaseClaim({ _tag: "rate_limited", outcome: "rate_limited" } as const);
1872
}
1628
- if (!signatureVerified) {
1629
- yield* log("rejected_signature");
1630
- yield* observe("rejected_signature");
1631
- return { _tag: "rejected_signature" as const };
1873
+ const reserve = takeRateSlot(task.id, DateTime.toEpochMillis(now)).pipe(
1874
+ Effect.flatMap((slot) => (slot === "allowed" ? Effect.void : Effect.fail(slot))),
1875
+ );
1876
+ const queueKey = `${task.id}\u0000${task.createdAt}`;
1877
+ const checkCapacity = Effect.gen(function* () {
1878
+ const [row] = yield* sql<{
1879
+ count: number;
1880
+ }>`SELECT COUNT(*) AS count FROM scheduled_task_webhook_pending
1881
+ WHERE task_id = ${task.id} AND state IN ('pending', 'dispatching', 'running')`;
1882
+ if (
1883
+ (row?.count ?? 0) + ((yield* Ref.get(webhookQueued)).get(queueKey) ?? 0) >=
1884
+ WEBHOOK_MAX_QUEUED_PER_TASK
1885
+ )
1886
+ return yield* Effect.fail("queue_full" as const);
1887
+ }).pipe(
1888
+ Effect.mapError((cause) => (cause === "queue_full" ? cause : dedupeError(cause))),
1889
+ );
1890
+ const admit = <A, E>(effect: Effect.Effect<A, E>) =>
1891
+ sql.withTransaction(effect).pipe(
1892
+ Effect.catch((cause) =>
1893
+ cause === "first_rejected" || cause === "rejected" || cause === "queue_full"
1894
+ ? Effect.succeed({
1895
+ outcome:
1896
+ cause === "queue_full"
1897
+ ? ("queue_full" as const)
1898
+ : ("rate_limited" as const),
1899
+ log: cause === "first_rejected",
1900
+ })
1901
+ : Effect.fail(isScheduledTaskError(cause) ? cause : dedupeError(cause)),
1902
+ ),
1903
+ );
1904
+ const deduped =
1905
+ schedule.dedupe === undefined
1906
+ ? "run"
1907
+ : yield* admit(
1908
+ Effect.gen(function* () {
1909
+ const outcome = yield* acceptWebhookDedupe(
1910
+ task,
1911
+ request,
1912
+ deliveryId,
1913
+ DateTime.toEpochMillis(now),
1914
+ reserve,
1915
+ takeDedupeLogSlot(task.id, DateTime.toEpochMillis(now)),
1916
+ checkCapacity,
1917
+ (ownerId) => replaceDeliveryRequest(ownerId, request),
1918
+ ).pipe(Effect.provideService(SqlClient.SqlClient, sql));
1919
+ if (
1920
+ outcome === "pending" ||
1921
+ outcome === "coalesced" ||
1922
+ (outcome === "duplicate" &&
1923
+ (yield* takeDedupeLogSlot(task.id, DateTime.toEpochMillis(now))))
1924
+ )
1925
+ yield* log(outcome === "pending" ? "accepted" : outcome, {
1926
+ signatureVerified: signature !== null,
1927
+ });
1928
+ return outcome;
1929
+ }),
1930
+ );
1931
+ if (typeof deduped !== "string") {
1932
+ yield* refundRejectionSlot;
1933
+ if (deduped.log) yield* log("rate_limited");
1934
+ yield* observe(deduped.outcome);
1935
+ return yield* releaseClaim({ _tag: "rate_limited", outcome: deduped.outcome } as const);
1936
}
1633
- if (tooOld) {
1634
- yield* log("expired", { signatureVerified: signature !== null });
1635
- yield* observe("expired");
1636
- return { _tag: "expired" as const };
1937
+ if (deduped !== "run") {
1938
+ if (deduped !== "coalesced" && deduped !== "coalesced_unlogged")
1939
+ yield* refundRejectionSlot;
1940
+ const outcome =
1941
+ deduped === "pending"
1942
+ ? "accepted"
1943
+ : deduped === "coalesced_unlogged"
1944
+ ? "coalesced"
1945
+ : deduped;
1946
+ yield* observe(outcome);
1947
+ if (deduped === "duplicate")
1948
+ return yield* releaseClaim({ _tag: "accepted", deliveryId, outcome } as const);
1949
+ claimAcquired = false;
1950
+ return { _tag: "accepted", deliveryId, outcome } as const;
1951
}
1638
-
1952
const frameId = yield* crypto.randomBytes(6).pipe(
1953
Effect.map((bytes) => Buffer.from(bytes).toString("hex")),
1954
Effect.mapError((cause) => taskError("Could not generate frame id.", { cause })),
@@ -1648,85 +1961,192 @@ export const layer = Layer.effect(
1961
signatureVerified: signature !== null,
1962
missing: rendered.missing,
1963
renderedPrompt: rendered.prompt,
1651
- error: "The filled-in prompt is too long.",
1964
+ error: PROMPT_TOO_LONG,
1965
});
1966
+ yield* releaseReplay(deliveryId);
1967
yield* observe("prompt_too_long");
1654
- return { _tag: "accepted" as const, deliveryId, outcome: "prompt_too_long" as const };
1968
+ return yield* releaseClaim({
1969
+ _tag: "accepted" as const,
1970
+ deliveryId,
1971
+ outcome: "prompt_too_long" as const,
1972
+ });
1973
}
1974
// Bound the deliveries one task holds, so steady traffic to a stuck
1975
// task cannot pile up parked fibers.
1658
- const queueKey = `${task.id}\u0000${task.createdAt}`;
1659
- const queued = yield* Ref.modify(webhookQueued, (counts) => {
1660
- const count = counts.get(queueKey) ?? 0;
1661
- return count >= WEBHOOK_MAX_QUEUED_PER_TASK
1662
- ? ([false, counts] as const)
1663
- : ([true, new Map(counts).set(queueKey, count + 1)] as const);
1976
+ yield* refundRejectionSlot;
1977
+ let queueReserved = false;
1978
+ const release = Effect.suspend(() => {
1979
+ if (!queueReserved) return Effect.void;
1980
+ queueReserved = false;
1981
+ return Ref.update(webhookQueued, (counts) => {
1982
+ const next = new Map(counts);
1983
+ const count = (next.get(queueKey) ?? 1) - 1;
1984
+ if (count <= 0) next.delete(queueKey);
1985
+ else next.set(queueKey, count);
1986
+ return next;
1987
+ });
1988
});
1665
- if (!queued) {
1666
- yield* observe("queue_full");
1989
+ // Serialize immediate reservations with persisted window admission.
1990
+ const reserveImmediate = Effect.gen(function* () {
1991
+ yield* checkCapacity;
1992
+ yield* reserve;
1993
+ yield* Ref.update(webhookQueued, (counts) =>
1994
+ new Map(counts).set(queueKey, (counts.get(queueKey) ?? 0) + 1),
1995
+ );
1996
+ queueReserved = true;
1997
+ });
1998
+ const admitted = yield* admit(
1999
+ Effect.gen(function* () {
2000
+ const outcome = yield* acceptImmediateWebhookDedupe(
2001
+ task,
2002
+ request,
2003
+ deliveryId,
2004
+ DateTime.toEpochMillis(now),
2005
+ reserveImmediate,
2006
+ ).pipe(Effect.provideService(SqlClient.SqlClient, sql));
2007
+ if (outcome === "duplicate") {
2008
+ if (yield* takeDedupeLogSlot(task.id, DateTime.toEpochMillis(now)))
2009
+ yield* log("duplicate", { signatureVerified: signature !== null });
2010
+ } else
2011
+ yield* log("accepted", {
2012
+ signatureVerified: signature !== null,
2013
+ missing: rendered.missing,
2014
+ renderedPrompt: rendered.prompt,
2015
+ });
2016
+ return outcome;
2017
+ }),
2018
+ ).pipe(Effect.onError(() => release));
2019
+ if (typeof admitted !== "string") {
2020
+ yield* release;
2021
+ if (admitted.log) yield* log("rate_limited");
2022
+ yield* observe(admitted.outcome);
2023
return yield* releaseClaim({
1668
- _tag: "rate_limited" as const,
1669
- outcome: "queue_full" as const,
1670
- });
2024
+ _tag: "rate_limited",
2025
+ outcome: admitted.outcome,
2026
+ } as const);
2027
+ }
2028
+ if (admitted === "duplicate") {
2029
+ yield* release;
2030
+ yield* observe("duplicate");
2031
+ return yield* releaseClaim({
2032
+ _tag: "accepted",
2033
+ deliveryId,
2034
+ outcome: "duplicate",
2035
+ } as const);
2036
}
1672
- const release = Ref.update(webhookQueued, (counts) => {
1673
- const next = new Map(counts);
1674
- const count = (next.get(queueKey) ?? 1) - 1;
1675
- if (count <= 0) next.delete(queueKey);
1676
- else next.set(queueKey, count);
1677
- return next;
1678
- });
1679
- yield* log("accepted", {
1680
- signatureVerified: signature !== null,
1681
- missing: rendered.missing,
1682
- renderedPrompt: rendered.prompt,
1683
- }).pipe(Effect.onError(() => release));
2037
yield* observe("accepted");
2038
const permit = yield* webhookPermit(task.id);
1686
- const runOutcome = (outcome: "started" | "skipped" | "failed") =>
1687
- Effect.all([
1688
- Effect.annotateCurrentSpan({ "scheduled_task.webhook.run_outcome": outcome }),
1689
- Metrics.increment(Metrics.webhookRunsTotal, { outcome }),
1690
- ]);
1691
- yield* runTask(task, "webhook", { deliveryId, prompt: rendered.prompt }).pipe(
1692
- Effect.flatMap(({ task: completed }) =>
1693
- completed.lastRunStatus === "failed"
1694
- ? runOutcome("failed").pipe(
1695
- Effect.andThen(markDeliveryFailed(deliveryId, "The run failed to start.")),
1696
- )
1697
- : runOutcome("started"),
1698
- ),
1699
- Effect.catchTags({
1700
- WebhookDeliverySkipped: (skipped) =>
1701
- runOutcome("skipped").pipe(
1702
- Effect.andThen(markDeliveryFailed(deliveryId, skipped.reason)),
1703
- ),
1704
- }),
1705
- // The log is readable over RPC, so it gets a fixed reason; the
1706
- // cause, which can carry request data, stays in the server log.
1707
- Effect.catchCause((cause) =>
1708
- Effect.logWarning("Webhook dispatch failed", { taskId: task.id, cause }).pipe(
1709
- Effect.andThen(runOutcome("failed")),
1710
- Effect.andThen(markDeliveryFailed(deliveryId, "The run failed to start.")),
1711
- ),
1712
- ),
2039
+ yield* runWebhookDelivery(task, deliveryId, rendered.prompt).pipe(
2040
permit.withPermits(1),
1714
- Effect.withSpan("ScheduledTaskService.runWebhookDelivery", {
1715
- root: true,
1716
- attributes: {
1717
- "scheduled_task.id": task.id,
1718
- "scheduled_task.webhook.delivery_id": deliveryId,
1719
- },
1720
- }),
2041
Effect.ensuring(release),
2042
Effect.forkIn(serviceScope),
2043
);
2044
// Dispatch owns the accepted delivery now; retries must keep its claim.
2045
claimAcquired = false;
2046
return { _tag: "accepted" as const, deliveryId, outcome: "accepted" as const };
1727
- }).pipe(Effect.onError(() => releaseClaim(undefined))),
2047
+ }).pipe(
2048
+ Effect.onError(() =>
2049
+ Effect.all([
2050
+ releaseClaim(undefined),
2051
+ (request.relayDeliveryId === undefined || claimAcquired
2052
+ ? releaseReplay(deliveryId)
2053
+ : Effect.void
2054
+ ).pipe(Effect.ignore),
2055
+ ]),
2056
+ ),
2057
+ ),
2058
+ );
2059
+ });
2060
+
2061
+ const dispatchPending = Effect.fn("ScheduledTaskService.dispatchPendingWebhooks")(function* () {
2062
+ const now = DateTime.toEpochMillis(yield* localNow);
2063
+ yield* sql`UPDATE scheduled_task_webhook_pending SET state = 'dispatching' WHERE state = 'pending' AND due_at <= ${now}`;
2064
+ const tasks = yield* sql<{ task_id: ScheduledTaskId }>`
2065
+ SELECT DISTINCT task_id FROM scheduled_task_webhook_pending WHERE state IN ('dispatching', 'running', 'completed', 'failed')
2066
+ `;
2067
+ yield* Effect.forEach(
2068
+ tasks,
2069
+ ({ task_id: taskId }) =>
2070
+ Effect.gen(function* () {
2071
+ const reserved = yield* Ref.modify(pendingInFlight, (active) =>
2072
+ active.has(taskId)
2073
+ ? ([false, active] as const)
2074
+ : ([true, new Set(active).add(taskId)] as const),
2075
+ );
2076
+ if (!reserved) return;
2077
+ const permit = yield* webhookPermit(taskId);
2078
+ // One worker per task waits on its permit, then drains persisted rows in order.
2079
+ // Other tasks remain independent and ticks never reload an active task's payloads.
2080
+ const dispatch = Effect.gen(function* () {
2081
+ while (true) {
2082
+ const [entry] = yield* sql<{
2083
+ delivery_id: string;
2084
+ task_json: string;
2085
+ request_json: string;
2086
+ state: string;
2087
+ }>`
2088
+ SELECT delivery_id, task_json, request_json, state FROM scheduled_task_webhook_pending
2089
+ WHERE task_id = ${taskId} AND state IN ('dispatching', 'running', 'completed', 'failed') ORDER BY due_at, rowid LIMIT 1
2090
+ `;
2091
+ if (entry === undefined) return;
2092
+ const remove = sql`DELETE FROM scheduled_task_webhook_pending WHERE delivery_id = ${entry.delivery_id}`;
2093
+ if (entry.state === "completed" || entry.state === "failed") {
2094
+ if (entry.state === "failed") {
2095
+ yield* markDeliveryFailed(entry.delivery_id, "The run failed to start.");
2096
+ yield* releaseFailedDelivery(entry.delivery_id);
2097
+ }
2098
+ yield* remove;
2099
+ continue;
2100
+ }
2101
+ const prepared = yield* Effect.gen(function* () {
2102
+ const task = yield* decodePendingTask(entry.task_json);
2103
+ const request = yield* decodePendingRequest(entry.request_json);
2104
+ const frameId = Buffer.from(yield* crypto.randomBytes(6)).toString("hex");
2105
+ return { task, rendered: renderWebhookPrompt(task.prompt, request, frameId) };
2106
+ }).pipe(Effect.exit);
2107
+ if (Exit.isFailure(prepared)) {
2108
+ if (Cause.hasInterruptsOnly(prepared.cause))
2109
+ return yield* Effect.failCause(prepared.cause);
2110
+ yield* markDeliveryFailed(
2111
+ entry.delivery_id,
2112
+ "The pending webhook payload could not be prepared.",
2113
+ );
2114
+ yield* releaseFailedDelivery(entry.delivery_id);
2115
+ yield* remove;
2116
+ continue;
2117
+ }
2118
+ const { task, rendered } = prepared.value;
2119
+ if (rendered.prompt.trim().length > PROVIDER_SEND_TURN_MAX_INPUT_CHARS) {
2120
+ yield* markDeliveryFailed(entry.delivery_id, PROMPT_TOO_LONG);
2121
+ yield* releaseFailedDelivery(entry.delivery_id);
2122
+ } else {
2123
+ yield* sql`UPDATE scheduled_task_webhook_deliveries SET outcome = 'accepted', error = NULL, rendered_prompt = ${rendered.prompt.slice(0, WEBHOOK_DELIVERY_LOG_PROMPT_LIMIT)}, missing_fields_json = ${encodeMissingFieldsJson(rendered.missing)} WHERE delivery_id = ${entry.delivery_id}`;
2124
+ if (!(yield* runWebhookDelivery(task, entry.delivery_id, rendered.prompt)))
2125
+ return;
2126
+ }
2127
+ yield* remove;
2128
+ }
2129
+ });
2130
+ yield* dispatch.pipe(
2131
+ permit.withPermits(1),
2132
+ Effect.catch((cause) =>
2133
+ Effect.logWarning("Pending webhook dispatch failed", { cause }),
2134
+ ),
2135
+ Effect.ensuring(
2136
+ Ref.update(pendingInFlight, (active) => {
2137
+ const next = new Set(active);
2138
+ next.delete(taskId);
2139
+ return next;
2140
+ }),
2141
+ ),
2142
+ Effect.withSpan("ScheduledTaskService.dispatchPendingWebhook", { root: true }),
2143
+ Effect.forkIn(serviceScope),
2144
+ );
2145
+ }),
2146
+ { discard: true },
2147
);
2148
});
2149
+ yield* scheduler.register("webhook-coalescing", dispatchPending());
2150
2151
return ScheduledTaskService.of({
2152
list,
apps/server/src/scheduledTasks/ScheduledTaskService.webhook.test.ts
+1903
-125
@@ -3,8 +3,13 @@ import * as NodeCrypto from "node:crypto";
3
4
import * as NodePlatformCrypto from "@effect/platform-node/NodeCrypto";
5
import { assert, it } from "@effect/vitest";
6
-import { ScheduledTaskUpsertInput, SecretRequestError } from "@t3tools/contracts";
6
+import {
7
+ ScheduledTaskUpsertInput,
8
+ ScheduledTaskWebhookDeliveryId,
9
+ SecretRequestError,
10
+} from "@t3tools/contracts";
11
import * as DateTime from "effect/DateTime";
12
+import * as Crypto from "effect/Crypto";
13
import * as Deferred from "effect/Deferred";
14
import * as Effect from "effect/Effect";
15
import * as Fiber from "effect/Fiber";
@@ -24,9 +29,94 @@ import * as ManagedProjectFolders from "../project/ManagedProjectFolders.ts";
29
import * as SqlitePersistence from "../persistence/Sqlite.ts";
30
import * as Scheduler from "../scheduling/Scheduler.ts";
31
import * as ScheduledTaskService from "./ScheduledTaskService.ts";
32
+import { renderWebhookPrompt } from "./webhookTemplate.ts";
33
34
const decodeUpsertInput = Schema.decodeUnknownEffect(ScheduledTaskUpsertInput);
35
36
+it.effect(
37
+ "preserves omitted dedupe settings, releases duplicate claims, and retains coalesced claims",
38
+ () =>
39
+ withService(({ service, sql, launches }) =>
40
+ Effect.gen(function* () {
41
+ const dedupe = {
42
+ replayKey: "{{headers.delivery-id}}",
43
+ coalesceKey: "{{body.group}}",
44
+ coalesceWindowSeconds: 10,
45
+ };
46
+ const { task } = yield* service.upsert(
47
+ yield* webhookTaskInput({ schedule: { type: "webhook", dedupe } }),
48
+ );
49
+ const updated = yield* service.upsert(yield* webhookTaskInput());
50
+ assert.deepEqual(
51
+ updated.task.schedule.type === "webhook" && updated.task.schedule.dedupe,
52
+ dedupe,
53
+ );
54
+ const request = (id: string, bodyText = "{}", relayDeliveryId?: string) =>
55
+ requestFor(task, {
56
+ headers: { "delivery-id": id },
57
+ bodyText,
58
+ body: new TextEncoder().encode(bodyText),
59
+ ...(relayDeliveryId === undefined ? {} : { relayDeliveryId }),
60
+ });
61
+ yield* service.triggerWebhook(request("A"));
62
+ yield* Queue.take(launches);
63
+ yield* service.triggerWebhook(request("B1", '{"group":"B"}'));
64
+ for (let i = 0; i < 5; i++) {
65
+ const duplicate = yield* service.triggerWebhook(request("A", "{}", `duplicate-${i}`));
66
+ assert.equal(duplicate._tag === "accepted" && duplicate.outcome, "duplicate");
67
+ const coalesced = yield* service.triggerWebhook(
68
+ request(`B${i + 2}`, '{"group":"B"}', `coalesced-${i}`),
69
+ );
70
+ assert.equal(coalesced._tag === "accepted" && coalesced.outcome, "coalesced");
71
+ }
72
+ const claims = yield* sql<{
73
+ relay_delivery_id: string;
74
+ }>`SELECT relay_delivery_id FROM scheduled_task_webhook_relay_deliveries ORDER BY relay_delivery_id`;
75
+ assert.deepEqual(
76
+ claims.map((claim) => claim.relay_delivery_id),
77
+ Array.from({ length: 5 }, (_, i) => `coalesced-${i}`),
78
+ );
79
+ assert.isTrue(
80
+ (yield* service.listWebhookDeliveries({ id: task.id })).deliveries.every(
81
+ (delivery) => delivery.outcome !== "duplicate" && delivery.outcome !== "coalesced",
82
+ ),
83
+ );
84
+ const fresh = yield* service.triggerWebhook(request("C", "{}", "duplicate-0"));
85
+ assert.equal(fresh._tag === "accepted" && fresh.outcome, "accepted");
86
+ yield* Queue.take(launches);
87
+ const cleared = yield* service.upsert(
88
+ yield* webhookTaskInput({ schedule: { type: "webhook", dedupe: {} } }),
89
+ );
90
+ assert.deepEqual(
91
+ cleared.task.schedule.type === "webhook" && cleared.task.schedule.dedupe,
92
+ {},
93
+ );
94
+ }),
95
+ ),
96
+);
97
+
98
+const waitForSpan = <A, E, R>(work: Effect.Effect<A, E, R>, name: string) =>
99
+ Effect.gen(function* () {
100
+ const done = yield* Deferred.make<void>();
101
+ const tracer = Tracer.make({
102
+ span: (options) => {
103
+ const span = new Tracer.NativeSpan(options);
104
+ const end = span.end.bind(span);
105
+ span.end = (...args) => {
106
+ end(...args);
107
+ if (span.name === name) Deferred.doneUnsafe(done, Effect.void);
108
+ };
109
+ return span;
110
+ },
111
+ });
112
+ const result = yield* work.pipe(Effect.withTracer(tracer));
113
+ yield* Deferred.await(done);
114
+ return result;
115
+ });
116
+
117
+const drainPendingDispatch = (workers: Map<string, Effect.Effect<void>>) =>
118
+ waitForSpan(workers.get("webhook-coalescing")!, "ScheduledTaskService.dispatchPendingWebhook");
119
+
120
type LaunchInput = ThreadLaunchService.ThreadLaunchInput;
121
122
const webhookTaskInput = (overrides: Record<string, unknown> = {}) =>
@@ -70,20 +160,76 @@ const withService = <A, E>(
160
readonly sql: SqlClient.SqlClient;
161
readonly secretsByRef: Map<string, string>;
162
}) => Effect.Effect<A, E, never>,
73
- options: { readonly gate?: Deferred.Deferred<void>; readonly relayHookBaseUrl?: string } = {},
163
+ options: {
164
+ readonly onFrame?: () => void;
165
+ readonly frameGate?: {
166
+ readonly started: Deferred.Deferred<void>;
167
+ readonly release: Deferred.Deferred<void>;
168
+ };
169
+ readonly completedLaunches?: Set<string>;
170
+ readonly failLaunches?: { remaining: number };
171
+ readonly gate?: Deferred.Deferred<void>;
172
+ readonly gateForTask?: (input: LaunchInput) => Deferred.Deferred<void> | undefined;
173
+ readonly relayHookBaseUrl?: string;
174
+ readonly sql?: SqlClient.SqlClient;
175
+ readonly workers?: Map<string, Effect.Effect<void>>;
176
+ } = {},
177
) =>
178
Effect.gen(function* () {
179
const launches = yield* Queue.unbounded<LaunchInput>();
180
const secretsByRef = new Map<string, string>();
181
const layerDependencies = Layer.mergeAll(
79
- NodePlatformCrypto.layer,
80
- Scheduler.layer,
182
+ options.frameGate === undefined && options.onFrame === undefined
183
+ ? NodePlatformCrypto.layer
184
+ : Layer.effect(
185
+ Crypto.Crypto,
186
+ Effect.gen(function* () {
187
+ const crypto = yield* Crypto.Crypto;
188
+ return {
189
+ ...crypto,
190
+ randomBytes: (size: number) =>
191
+ Effect.suspend(() => {
192
+ if (size === 6) options.onFrame?.();
193
+ return size === 6 && options.frameGate !== undefined
194
+ ? Deferred.succeed(options.frameGate.started, undefined).pipe(
195
+ Effect.andThen(Deferred.await(options.frameGate.release)),
196
+ Effect.andThen(crypto.randomBytes(size)),
197
+ )
198
+ : crypto.randomBytes(size);
199
+ }),
200
+ };
201
+ }),
202
+ ).pipe(Layer.provide(NodePlatformCrypto.layer)),
203
+ options.workers === undefined
204
+ ? Scheduler.layer
205
+ : Layer.succeed(Scheduler.Scheduler, {
206
+ register: <E, R>(name: string, work: Effect.Effect<void, E, R>) =>
207
+ Effect.gen(function* () {
208
+ const context = yield* Effect.context<R>();
209
+ options.workers!.set(name, work.pipe(Effect.provideContext(context), Effect.orDie));
210
+ }),
211
+ }),
212
Layer.mock(ThreadLaunchService.ThreadLaunchService)({
213
launch: (input) =>
83
- Queue.offer(launches, input).pipe(
84
- Effect.andThen(options.gate ? Deferred.await(options.gate) : Effect.void),
85
- Effect.as({ threadId: "thread-1", resumed: false } as never),
86
- ),
214
+ Effect.suspend(() => {
215
+ if (options.completedLaunches?.has(input.commandId))
216
+ return Effect.succeed({ threadId: "thread-1", resumed: false } as never);
217
+ options.completedLaunches?.add(input.commandId);
218
+ return Queue.offer(launches, input).pipe(
219
+ Effect.andThen(
220
+ (options.gateForTask?.(input) ?? options.gate)
221
+ ? Deferred.await((options.gateForTask?.(input) ?? options.gate)!)
222
+ : Effect.void,
223
+ ),
224
+ Effect.andThen(
225
+ Effect.sync(() => {
226
+ if (options.failLaunches && options.failLaunches.remaining-- > 0)
227
+ throw new Error("injected start failure");
228
+ }),
229
+ ),
230
+ Effect.as({ threadId: "thread-1", resumed: false } as never),
231
+ );
232
+ }),
233
}),
234
Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({
235
namedProjectsRoot: "/projects",
@@ -108,8 +254,18 @@ const withService = <A, E>(
254
const service = yield* ScheduledTaskService.ScheduledTaskService;
255
const sql = yield* SqlClient.SqlClient;
256
return yield* body({ service, launches, sql, secretsByRef });
111
- }).pipe(Effect.provide(ScheduledTaskService.layer.pipe(Layer.provide(layerDependencies))));
112
- }).pipe(Effect.provide(SqlitePersistence.layerMemory));
257
+ }).pipe(
258
+ Effect.provide(
259
+ Layer.fresh(ScheduledTaskService.layer.pipe(Layer.provide(layerDependencies))),
260
+ ),
261
+ );
262
+ }).pipe(
263
+ Effect.provide(
264
+ options.sql === undefined
265
+ ? SqlitePersistence.layerMemory
266
+ : Layer.succeed(SqlClient.SqlClient, options.sql),
267
+ ),
268
+ );
269
270
it.effect("dispatches exactly the rendered prompt and logs the delivery", () =>
271
withService(({ service, launches }) =>
@@ -132,7 +288,10 @@ it.effect("dispatches exactly the rendered prompt and logs the delivery", () =>
288
`scheduled-task:${task.id}:webhook:${result._tag === "accepted" ? result.deliveryId : ""}`,
289
);
290
135
- const { deliveries } = yield* service.listWebhookDeliveries({ id: task.id });
291
+ const { deliveries } = yield* service.listWebhookDeliveries({
292
+ id: task.id,
293
+ acceptAnyOutcome: true,
294
+ });
295
assert.equal(deliveries.length, 1);
296
assert.equal(deliveries[0]?.outcome, "accepted");
297
const { delivery } = yield* service.getWebhookDelivery({
@@ -188,7 +347,11 @@ it.effect("answers not found for a wrong token or unknown hook without logging",
347
assert.equal(wrongToken._tag, "not_found");
348
const unknown = yield* service.triggerWebhook(requestFor(task, { hookId: "missing" }));
349
assert.equal(unknown._tag, "not_found");
191
- assert.equal((yield* service.listWebhookDeliveries({ id: task.id })).deliveries.length, 0);
350
+ assert.equal(
351
+ (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true })).deliveries
352
+ .length,
353
+ 0,
354
+ );
355
}),
356
),
357
);
@@ -253,9 +416,10 @@ it.effect("checks the configured signature and keeps the secret write-only", ()
416
);
417
assert.isTrue(resaved.task.webhook?.hasSecret);
418
256
- const outcomes = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries.map(
257
- (delivery) => [delivery.outcome, delivery.signatureVerified],
258
- );
419
+ const outcomes = (yield* service.listWebhookDeliveries({
420
+ id: task.id,
421
+ acceptAnyOutcome: true,
422
+ })).deliveries.map((delivery) => [delivery.outcome, delivery.signatureVerified]);
423
assert.deepEqual(outcomes.toSorted(), [
424
["accepted", true],
425
["rejected_signature", false],
@@ -300,23 +464,38 @@ it.effect("queues a burst of deliveries instead of dropping them", () =>
464
}),
465
);
466
303
-it.effect("rate limits a hook past 60 deliveries a minute", () =>
304
- withService(({ service }) =>
305
- Effect.gen(function* () {
306
- const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
307
- const results = yield* Effect.forEach(Array.from({ length: 61 }), () =>
308
- service.triggerWebhook(requestFor(task)),
309
- );
310
- assert.equal(results.at(-2)?._tag, "disabled");
311
- assert.equal(results.at(-1)?._tag, "rate_limited");
312
- yield* Effect.forEach([1, 2, 3], () => service.triggerWebhook(requestFor(task)));
313
- const outcomes = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries.map(
314
- (delivery) => delivery.outcome,
315
- );
316
- assert.equal(outcomes.filter((outcome) => outcome === "rate_limited").length, 1);
317
- }),
318
- ),
319
-);
467
+it.effect("rate limits a hook past 60 deliveries a minute", () => {
468
+ let frames = 0;
469
+ return withService(
470
+ ({ service, launches }) =>
471
+ Effect.gen(function* () {
472
+ const { task } = yield* service.upsert(yield* webhookTaskInput());
473
+ const results = yield* Effect.forEach(Array.from({ length: 61 }), () =>
474
+ service
475
+ .triggerWebhook(requestFor(task))
476
+ .pipe(
477
+ Effect.tap((result) =>
478
+ result._tag === "accepted" ? Queue.take(launches) : Effect.void,
479
+ ),
480
+ ),
481
+ );
482
+ assert.equal(results.at(-2)?._tag, "accepted");
483
+ assert.equal(results.at(-1)?._tag, "rate_limited");
484
+ yield* Effect.forEach([1, 2, 3], () => service.triggerWebhook(requestFor(task)));
485
+ const outcomes = (yield* service.listWebhookDeliveries({
486
+ id: task.id,
487
+ acceptAnyOutcome: true,
488
+ })).deliveries.map((delivery) => delivery.outcome);
489
+ assert.equal(outcomes.filter((outcome) => outcome === "rate_limited").length, 1);
490
+ assert.equal(frames, 60);
491
+ }),
492
+ {
493
+ onFrame: () => {
494
+ frames++;
495
+ },
496
+ },
497
+ );
498
+});
499
500
it.effect("a save carrying a stale token cannot undo a rotation", () =>
501
withService(({ service }) =>
@@ -337,7 +516,10 @@ it.effect("keeps the newest 50 deliveries when they share a timestamp", () =>
516
const ids = yield* Effect.forEach(Array.from({ length: 55 }), (_, index) =>
517
service.triggerWebhook(requestFor(task, { query: `n=${index}` })).pipe(Effect.as(index)),
518
);
340
- const { deliveries } = yield* service.listWebhookDeliveries({ id: task.id });
519
+ const { deliveries } = yield* service.listWebhookDeliveries({
520
+ id: task.id,
521
+ acceptAnyOutcome: true,
522
+ });
523
assert.equal(deliveries.length, 50);
524
const first = yield* service.getWebhookDelivery({
525
id: task.id,
@@ -368,7 +550,10 @@ it.effect("keeps credential headers and query values out of the delivery log", (
550
},
551
}),
552
);
371
- const [summary] = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries;
553
+ const [summary] = (yield* service.listWebhookDeliveries({
554
+ id: task.id,
555
+ acceptAnyOutcome: true,
556
+ })).deliveries;
557
const { delivery } = yield* service.getWebhookDelivery({
558
id: task.id,
559
deliveryId: summary!.id,
@@ -479,7 +664,10 @@ it.effect("caps deliveries waiting behind a stuck run", () =>
664
);
665
assert.equal(waiting.filter((result) => result._tag === "accepted").length, 19);
666
assert.equal(waiting.at(-1)?._tag, "rate_limited");
482
- const logged = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries;
667
+ const logged = (yield* service.listWebhookDeliveries({
668
+ id: task.id,
669
+ acceptAnyOutcome: true,
670
+ })).deliveries;
671
assert.equal(logged.length, 20);
672
yield* Deferred.succeed(gate, undefined);
673
}),
@@ -495,7 +683,10 @@ it.effect("logs a body's first 64 KiB by bytes, not characters", () =>
683
const text = "界".repeat(30_000);
684
const body = new TextEncoder().encode(text);
685
yield* service.triggerWebhook(requestFor(task, { body, bodyText: text }));
498
- const [summary] = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries;
686
+ const [summary] = (yield* service.listWebhookDeliveries({
687
+ id: task.id,
688
+ acceptAnyOutcome: true,
689
+ })).deliveries;
690
const { delivery } = yield* service.getWebhookDelivery({
691
id: task.id,
692
deliveryId: summary!.id,
@@ -559,7 +750,8 @@ it.effect("a held request already delivered directly runs only once", () =>
750
assert.equal(direct._tag, "accepted");
751
assert.deepEqual(replayed, { ...direct, outcome: "duplicate" } as typeof replayed);
752
yield* Queue.take(launches);
562
- const logged = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries;
753
+ const logged = (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true }))
754
+ .deliveries;
755
assert.equal(logged.length, 1);
756
}),
757
),
@@ -580,7 +772,9 @@ it.effect("relay retries recover failed acceptance without releasing another req
772
const error = yield* service.triggerWebhook(request).pipe(Effect.flip);
773
assert.equal(error.message, "Could not record webhook delivery.");
774
assert.equal(yield* Queue.size(launches), 0);
583
- assert.isEmpty((yield* service.listWebhookDeliveries({ id: task.id })).deliveries);
775
+ assert.isEmpty(
776
+ (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true })).deliveries,
777
+ );
778
yield* sql`DROP TRIGGER fail_webhook_delivery_log`;
779
780
const retried = yield* Effect.all(
@@ -606,7 +800,11 @@ it.effect("relay retries recover failed acceptance without releasing another req
800
const replayed = yield* service.triggerWebhook(request);
801
assert.equal(replayed._tag === "accepted" ? replayed.outcome : replayed._tag, "duplicate");
802
assert.equal(yield* Queue.size(launches), 0);
609
- assert.equal((yield* service.listWebhookDeliveries({ id: task.id })).deliveries.length, 1);
803
+ assert.equal(
804
+ (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true })).deliveries
805
+ .length,
806
+ 1,
807
+ );
808
}),
809
),
810
);
@@ -657,9 +855,9 @@ it.effect("a held request runs once even after the log has trimmed it", () =>
855
it.effect("a rate-limited held request can run on a later pass", () =>
856
withService(({ service, launches }) =>
857
Effect.gen(function* () {
660
- const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
858
+ const { task } = yield* service.upsert(yield* webhookTaskInput());
859
yield* Effect.forEach(Array.from({ length: 60 }), () =>
662
- service.triggerWebhook(requestFor(task)),
860
+ service.triggerWebhook(requestFor(task)).pipe(Effect.tap(() => Queue.take(launches))),
861
);
862
const limited = yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "later" }));
863
assert.equal(limited._tag, "rate_limited");
@@ -678,7 +876,10 @@ it.effect("logs a held request at the time the relay received it", () =>
876
const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
877
const receivedAt = DateTime.formatIso(DateTime.subtract(yield* DateTime.now, { minutes: 5 }));
878
yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "relay-2", receivedAt }));
681
- const [delivery] = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries;
879
+ const [delivery] = (yield* service.listWebhookDeliveries({
880
+ id: task.id,
881
+ acceptAnyOutcome: true,
882
+ })).deliveries;
883
assert.equal(delivery?.receivedAt, receivedAt);
884
}),
885
),
@@ -699,7 +900,10 @@ it.effect("a receive time in the future counts as now", () =>
900
);
901
assert.equal(result._tag, "accepted");
902
yield* Queue.take(launches);
702
- const [delivery] = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries;
903
+ const [delivery] = (yield* service.listWebhookDeliveries({
904
+ id: task.id,
905
+ acceptAnyOutcome: true,
906
+ })).deliveries;
907
assert.equal(delivery?.receivedAt, DateTime.formatIso(now));
908
}),
909
),
@@ -723,9 +927,10 @@ it.effect("skips a held request older than the task's max age", () =>
927
requestFor(task, { relayDeliveryId: "fresh", receivedAt: fresh }),
928
);
929
assert.equal(ok._tag, "accepted");
726
- const outcomes = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries.map(
727
- (delivery) => delivery.outcome,
728
- );
930
+ const outcomes = (yield* service.listWebhookDeliveries({
931
+ id: task.id,
932
+ acceptAnyOutcome: true,
933
+ })).deliveries.map((delivery) => delivery.outcome);
934
assert.includeMembers(outcomes, ["expired", "accepted"]);
935
}),
936
),
@@ -753,7 +958,11 @@ it.effect("deleting a task removes its delivery log", () =>
958
const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
959
yield* service.triggerWebhook(requestFor(task));
960
yield* service.delete({ id: task.id });
756
- assert.equal((yield* service.listWebhookDeliveries({ id: task.id })).deliveries.length, 0);
961
+ assert.equal(
962
+ (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true })).deliveries
963
+ .length,
964
+ 0,
965
+ );
966
}),
967
),
968
);
@@ -954,105 +1163,1602 @@ it.effect("counts each handled request by what happened to it", () =>
1163
),
1164
);
1165
957
-it.effect("checks filters after the signature and max age, and runs only matching deliveries", () =>
1166
+it.effect("dedupes replay keys per task, persists them, and expires them after seven days", () =>
1167
+ withService(
1168
+ ({ service, launches, sql }) =>
1169
+ Effect.gen(function* () {
1170
+ const schedule = { type: "webhook", dedupe: { replayKey: "{{headers.delivery-id}}" } };
1171
+ const { task } = yield* service.upsert(yield* webhookTaskInput({ schedule }));
1172
+ const request = requestFor(task, { headers: { "delivery-id": "A" } });
1173
+ const first = yield* service.triggerWebhook(request);
1174
+ assert.equal(first._tag === "accepted" && first.outcome, "accepted");
1175
+ yield* Queue.take(launches);
1176
+ const replay = yield* service.triggerWebhook(request);
1177
+ assert.equal(replay._tag === "accepted" && replay.outcome, "duplicate");
1178
+ assert.equal(yield* Queue.size(launches), 0);
1179
+ const other = yield* service.upsert(
1180
+ yield* webhookTaskInput({ id: "scheduled-task:other", schedule }),
1181
+ );
1182
+ yield* service.triggerWebhook(requestFor(other.task, { headers: request.headers }));
1183
+ yield* Queue.take(launches);
1184
+ yield* withService(
1185
+ ({ service: restarted }) =>
1186
+ Effect.gen(function* () {
1187
+ const replay = yield* restarted.triggerWebhook(request);
1188
+ assert.equal(replay._tag === "accepted" && replay.outcome, "duplicate");
1189
+ }),
1190
+ { sql, workers: new Map() },
1191
+ );
1192
+ yield* TestClock.adjust("7 days");
1193
+ const expired = yield* service.triggerWebhook(request);
1194
+ assert.equal(expired._tag === "accepted" && expired.outcome, "accepted");
1195
+ yield* Queue.take(launches);
1196
+ assert.include(
1197
+ (yield* service.listWebhookDeliveries({
1198
+ id: task.id,
1199
+ acceptAnyOutcome: true,
1200
+ })).deliveries.map((d) => d.outcome),
1201
+ "duplicate",
1202
+ );
1203
+ }),
1204
+ { workers: new Map() },
1205
+ ),
1206
+);
1207
+
1208
+it.effect.each([1, 3])(
1209
+ "coalesces %i deliveries into one run with the latest body, headers and query",
1210
+ (count) => {
1211
+ const workers = new Map<string, Effect.Effect<void>>();
1212
+ return withService(
1213
+ ({ service, launches }) =>
1214
+ Effect.gen(function* () {
1215
+ const { task } = yield* service.upsert(
1216
+ yield* webhookTaskInput({
1217
+ prompt: "{{body.value}} {{headers.version}} {{query.version}}",
1218
+ schedule: {
1219
+ type: "webhook",
1220
+ dedupe: { coalesceKey: "{{body.key}}", coalesceWindowSeconds: 10 },
1221
+ },
1222
+ }),
1223
+ );
1224
+ for (let value = 1; value <= count; value++) {
1225
+ const bodyText = JSON.stringify({ key: "B", value });
1226
+ yield* service.triggerWebhook(
1227
+ requestFor(task, {
1228
+ bodyText,
1229
+ body: new TextEncoder().encode(bodyText),
1230
+ headers: { version: String(value) },
1231
+ query: `version=${value}`,
1232
+ }),
1233
+ );
1234
+ if (value < count) yield* TestClock.adjust("2 seconds");
1235
+ }
1236
+ const owner = (yield* service.listWebhookDeliveries({
1237
+ id: task.id,
1238
+ acceptAnyOutcome: true,
1239
+ })).deliveries.find((delivery) => delivery.outcome === "accepted")!;
1240
+ assert.isTrue(owner.pendingWindow);
1241
+ const beforeDispatch = (yield* service.getWebhookDelivery({
1242
+ id: task.id,
1243
+ deliveryId: owner.id,
1244
+ acceptAnyOutcome: true,
1245
+ })).delivery;
1246
+ assert.isTrue(beforeDispatch.pendingWindow);
1247
+ assert.equal(beforeDispatch.body, JSON.stringify({ key: "B", value: count }));
1248
+ assert.equal(beforeDispatch.headers.version, String(count));
1249
+ assert.equal(beforeDispatch.query, `version=${count}`);
1250
+ assert.equal(
1251
+ beforeDispatch.bodyBytes,
1252
+ new TextEncoder().encode(beforeDispatch.body).byteLength,
1253
+ );
1254
+ assert.equal(yield* Queue.size(launches), 0);
1255
+ yield* workers.get("webhook-coalescing")!;
1256
+ assert.equal(yield* Queue.size(launches), 0);
1257
+ yield* TestClock.adjust(count === 1 ? "10 seconds" : "6 seconds");
1258
+ yield* workers.get("webhook-coalescing")!;
1259
+ const launched = yield* Queue.take(launches);
1260
+ assert.equal(
1261
+ (launched.initialMessage!.text.match(new RegExp(`>${count}<`, "g")) ?? []).length,
1262
+ 3,
1263
+ );
1264
+ yield* workers.get("webhook-coalescing")!;
1265
+ assert.equal(yield* Queue.size(launches), 0);
1266
+ const outcomes = (yield* service.listWebhookDeliveries({
1267
+ id: task.id,
1268
+ acceptAnyOutcome: true,
1269
+ })).deliveries.map((d) => d.outcome);
1270
+ const afterDispatch = (yield* service.getWebhookDelivery({
1271
+ id: task.id,
1272
+ deliveryId: owner.id,
1273
+ acceptAnyOutcome: true,
1274
+ })).delivery;
1275
+ assert.isFalse(afterDispatch.pendingWindow);
1276
+ assert.equal(afterDispatch.body, beforeDispatch.body);
1277
+ assert.include(afterDispatch.renderedPrompt!, `>${count}<`);
1278
+ assert.equal(outcomes.filter((o) => o === "accepted").length, 1);
1279
+ assert.equal(outcomes.filter((o) => o === "coalesced").length, count - 1);
1280
+ }),
1281
+ { workers },
1282
+ );
1283
+ },
1284
+);
1285
+
1286
+it.effect("recovers a persisted coalescing window after restarting the service", () => {
1287
+ const workers = new Map<string, Effect.Effect<void>>();
1288
+ return withService(
1289
+ ({ service, sql }) =>
1290
+ Effect.gen(function* () {
1291
+ const { task } = yield* service.upsert(
1292
+ yield* webhookTaskInput({
1293
+ schedule: {
1294
+ type: "webhook",
1295
+ dedupe: { coalesceKey: "{{headers.content-type}}", coalesceWindowSeconds: 10 },
1296
+ },
1297
+ }),
1298
+ );
1299
+ yield* service.triggerWebhook(requestFor(task));
1300
+ yield* TestClock.adjust("4 seconds");
1301
+ const restartedWorkers = new Map<string, Effect.Effect<void>>();
1302
+ yield* withService(
1303
+ ({ launches }) =>
1304
+ Effect.gen(function* () {
1305
+ yield* restartedWorkers.get("webhook-coalescing")!;
1306
+ assert.equal(yield* Queue.size(launches), 0);
1307
+ yield* TestClock.adjust("6 seconds");
1308
+ yield* restartedWorkers.get("webhook-coalescing")!;
1309
+ yield* Queue.take(launches);
1310
+ yield* restartedWorkers.get("webhook-coalescing")!;
1311
+ assert.equal(yield* Queue.size(launches), 0);
1312
+ assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_pending`).length, 0);
1313
+ }),
1314
+ { sql, workers: restartedWorkers },
1315
+ );
1316
+ }),
1317
+ { workers },
1318
+ );
1319
+});
1320
+
1321
+it.effect("empty dedupe key renders run normally and malformed templates fail at save time", () =>
1322
withService(({ service, launches }) =>
1323
Effect.gen(function* () {
1324
const { task } = yield* service.upsert(
1325
yield* webhookTaskInput({
1326
schedule: {
1327
type: "webhook",
964
- signature: {
965
- header: "x-hub-signature-256",
966
- encoding: "hex",
967
- prefix: "sha256=",
968
- secret: "s3cret",
969
- },
970
- maxDeliveryAgeMinutes: 30,
971
- filters: [
972
- { path: "headers.x-github-event", op: "equals", value: "pull_request" },
973
- { path: "body.action", op: "in", value: ["opened", "reopened"] },
974
- ],
1328
+ dedupe: { replayKey: "{{headers.absent}}", coalesceKey: "{{body.absent}}" },
1329
},
1330
}),
1331
);
978
- const delivery = (
979
- event: string,
980
- action: string,
981
- options: { readonly signed?: boolean; readonly receivedAt?: string } = {},
982
- ) => {
983
- const body = new TextEncoder().encode(
984
- JSON.stringify({ action, pull_request: { url: "https://github.com/org/repo/pull/45" } }),
1332
+ for (let i = 0; i < 2; i++) {
1333
+ const result = yield* service.triggerWebhook(requestFor(task, { headers: { absent: "" } }));
1334
+ assert.equal(result._tag === "accepted" && result.outcome, "accepted");
1335
+ yield* Queue.take(launches);
1336
+ }
1337
+ for (const replayKey of ["literal", "{{body.bad"]) {
1338
+ const invalid = yield* webhookTaskInput({
1339
+ schedule: { type: "webhook", dedupe: { replayKey } },
1340
+ }).pipe(Effect.result);
1341
+ assert.equal(invalid._tag, "Failure");
1342
+ }
1343
+ for (const coalesceWindowSeconds of [9, 3601, 10.5]) {
1344
+ const result = yield* webhookTaskInput({
1345
+ schedule: { type: "webhook", dedupe: { coalesceWindowSeconds } },
1346
+ }).pipe(Effect.result);
1347
+ assert.equal(result._tag, "Failure");
1348
+ }
1349
+ }),
1350
+ ),
1351
+);
1352
+
1353
+it.effect("applies replay before coalescing and defaults the fixed window to 120 seconds", () => {
1354
+ const workers = new Map<string, Effect.Effect<void>>();
1355
+ return withService(
1356
+ ({ service, launches }) =>
1357
+ Effect.gen(function* () {
1358
+ const { task } = yield* service.upsert(
1359
+ yield* webhookTaskInput({
1360
+ prompt: "{{body.value}}",
1361
+ schedule: {
1362
+ type: "webhook",
1363
+ dedupe: { replayKey: "{{headers.delivery-id}}", coalesceKey: "{{body.group}}" },
1364
+ },
1365
+ }),
1366
);
986
- const signature = `sha256=${NodeCrypto.createHmac("sha256", "s3cret").update(body).digest("hex")}`;
987
- return requestFor(task, {
988
- headers: {
989
- "content-type": "application/json",
990
- "x-github-event": event,
991
- ...(options.signed === false ? {} : { "x-hub-signature-256": signature }),
992
- },
993
- body,
994
- bodyText: new TextDecoder().decode(body),
995
- ...(options.receivedAt === undefined
996
- ? {}
997
- : { relayDeliveryId: "old", receivedAt: options.receivedAt }),
998
- });
999
- };
1000
- const now = yield* DateTime.now;
1001
- const old = DateTime.formatIso(DateTime.subtract(now, { minutes: 31 }));
1367
+ const request = (id: string, value: string) => {
1368
+ const bodyText = JSON.stringify({ group: "B", value });
1369
+ return requestFor(task, {
1370
+ bodyText,
1371
+ body: new TextEncoder().encode(bodyText),
1372
+ headers: { "delivery-id": id },
1373
+ });
1374
+ };
1375
+ yield* service.triggerWebhook(request("B1", "first"));
1376
+ yield* service.triggerWebhook(request("B2", "latest"));
1377
+ const replay = yield* service.triggerWebhook(
1378
+ request("B2", "replay must not replace latest"),
1379
+ );
1380
+ assert.equal(replay._tag === "accepted" && replay.outcome, "duplicate");
1381
+ yield* TestClock.adjust("119 seconds");
1382
+ yield* workers.get("webhook-coalescing")!;
1383
+ assert.equal(yield* Queue.size(launches), 0);
1384
+ yield* TestClock.adjust("1 second");
1385
+ yield* workers.get("webhook-coalescing")!;
1386
+ assert.include((yield* Queue.take(launches)).initialMessage!.text, ">latest<");
1387
+ }),
1388
+ { workers },
1389
+ );
1390
+});
1391
1003
- // A request failing a filter is still refused first for its signature or age.
1004
- const unsigned = yield* service.triggerWebhook(delivery("push", "opened", { signed: false }));
1005
- assert.equal(unsigned._tag, "rejected_signature");
1006
- const tooOld = yield* service.triggerWebhook(delivery("push", "opened", { receivedAt: old }));
1007
- assert.equal(tooOld._tag, "expired");
1392
+it.effect("rolls back replay and pending state when recording a delivery fails", () => {
1393
+ const workers = new Map<string, Effect.Effect<void>>();
1394
+ return withService(
1395
+ ({ service, sql, launches }) =>
1396
+ Effect.gen(function* () {
1397
+ const { task } = yield* service.upsert(
1398
+ yield* webhookTaskInput({
1399
+ schedule: {
1400
+ type: "webhook",
1401
+ dedupe: {
1402
+ replayKey: "{{headers.content-type}}",
1403
+ coalesceKey: "{{headers.content-type}}",
1404
+ coalesceWindowSeconds: 10,
1405
+ },
1406
+ },
1407
+ }),
1408
+ );
1409
+ yield* sql`CREATE TRIGGER fail_dedupe_log BEFORE INSERT ON scheduled_task_webhook_deliveries BEGIN SELECT RAISE(FAIL, 'injected'); END`;
1410
+ yield* service.triggerWebhook(requestFor(task)).pipe(Effect.flip);
1411
+ assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_pending`).length, 0);
1412
+ assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_replay_keys`).length, 0);
1413
+ yield* sql`DROP TRIGGER fail_dedupe_log`;
1414
+ const retry = yield* service.triggerWebhook(requestFor(task));
1415
+ assert.equal(retry._tag === "accepted" && retry.outcome, "accepted");
1416
+ yield* TestClock.adjust("10 seconds");
1417
+ yield* workers.get("webhook-coalescing")!;
1418
+ yield* Queue.take(launches);
1419
+ }),
1420
+ { workers },
1421
+ );
1422
+});
1423
1009
- const wrongEvent = yield* service.triggerWebhook(delivery("push", "opened"));
1010
- const wrongAction = yield* service.triggerWebhook(delivery("pull_request", "closed"));
1011
- assert.deepInclude(wrongEvent, { _tag: "accepted", outcome: "filtered" });
1012
- assert.deepInclude(wrongAction, { _tag: "accepted", outcome: "filtered" });
1013
- assert.equal(yield* Queue.size(launches), 0);
1014
- const [listed] = (yield* service.list()).tasks;
1015
- assert.equal(listed?.runCount, 0);
1016
- assert.equal(listed?.lastRunStatus, "never");
1424
+it.effect("starts a new fixed window at the deadline without replacing the expired payload", () => {
1425
+ const workers = new Map<string, Effect.Effect<void>>();
1426
+ return withService(
1427
+ ({ service, launches }) =>
1428
+ Effect.gen(function* () {
1429
+ const { task } = yield* service.upsert(
1430
+ yield* webhookTaskInput({
1431
+ prompt: "{{body}}",
1432
+ schedule: {
1433
+ type: "webhook",
1434
+ dedupe: { coalesceKey: "{{headers.content-type}}", coalesceWindowSeconds: 10 },
1435
+ },
1436
+ }),
1437
+ );
1438
+ const request = (bodyText: string) =>
1439
+ requestFor(task, { bodyText, body: new TextEncoder().encode(bodyText) });
1440
+ yield* service.triggerWebhook(request("first"));
1441
+ yield* TestClock.adjust("10 seconds");
1442
+ yield* service.triggerWebhook(request("second"));
1443
+ yield* workers.get("webhook-coalescing")!;
1444
+ assert.include((yield* Queue.take(launches)).initialMessage!.text, ">first<");
1445
+ assert.equal(yield* Queue.size(launches), 0);
1446
+ yield* TestClock.adjust("10 seconds");
1447
+ yield* workers.get("webhook-coalescing")!;
1448
+ assert.include((yield* Queue.take(launches)).initialMessage!.text, ">second<");
1449
+ }),
1450
+ { workers },
1451
+ );
1452
+});
1453
1018
- const matched = yield* service.triggerWebhook(delivery("pull_request", "opened"));
1019
- assert.deepInclude(matched, { _tag: "accepted", outcome: "accepted" });
1020
- assert.include((yield* Queue.take(launches)).initialMessage?.text ?? "", "pull/45");
1454
+it.effect("pausing a pending task prevents dispatch and deletion clears its dedupe state", () => {
1455
+ const workers = new Map<string, Effect.Effect<void>>();
1456
+ return withService(
1457
+ ({ service, launches, sql }) =>
1458
+ Effect.gen(function* () {
1459
+ const { task } = yield* service.upsert(
1460
+ yield* webhookTaskInput({
1461
+ schedule: {
1462
+ type: "webhook",
1463
+ dedupe: {
1464
+ replayKey: "{{headers.content-type}}",
1465
+ coalesceKey: "{{headers.content-type}}",
1466
+ coalesceWindowSeconds: 10,
1467
+ },
1468
+ },
1469
+ }),
1470
+ );
1471
+ yield* service.triggerWebhook(requestFor(task));
1472
+ yield* service.setEnabled({ id: task.id, enabled: false });
1473
+ yield* TestClock.adjust("10 seconds");
1474
+ yield* drainPendingDispatch(workers);
1475
+ assert.equal(yield* Queue.size(launches), 0);
1476
+ assert.equal(
1477
+ (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true }))
1478
+ .deliveries[0]?.outcome,
1479
+ "dispatch_failed",
1480
+ );
1481
+ yield* service.delete({ id: task.id });
1482
+ assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_replay_keys`).length, 0);
1483
+ assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_pending`).length, 0);
1484
+ }),
1485
+ { workers },
1486
+ );
1487
+});
1488
1022
- const deliveries = (yield* service.listWebhookDeliveries({
1023
- id: task.id,
1024
- acceptAnyOutcome: true,
1025
- })).deliveries;
1026
- const filtered = deliveries
1027
- .filter((entry) => entry.outcome === "filtered")
1028
- .map((entry) => [entry.signatureVerified, entry.error]);
1029
- assert.sameDeepMembers(filtered, [
1030
- [true, "Filter 1 (headers.x-github-event) did not match."],
1031
- [true, "Filter 2 (body.action) did not match."],
1032
- ]);
1033
- }),
1034
- ),
1489
+it.effect(
1490
+ "caps duplicate writes at ten per minute without evicting runs or consuming their rate slots",
1491
+ () =>
1492
+ withService(({ service, launches, sql }) =>
1493
+ Effect.gen(function* () {
1494
+ const { task } = yield* service.upsert(
1495
+ yield* webhookTaskInput({
1496
+ schedule: { type: "webhook", dedupe: { replayKey: "{{body}}" } },
1497
+ }),
1498
+ );
1499
+ yield* service.triggerWebhook(requestFor(task));
1500
+ yield* Queue.take(launches);
1501
+ const before = yield* deliveriesCounted("duplicate", "direct");
1502
+ for (let i = 0; i < 75; i++) {
1503
+ const result = yield* service.triggerWebhook(requestFor(task));
1504
+ assert.equal(result._tag === "accepted" && result.outcome, "duplicate");
1505
+ }
1506
+ assert.equal((yield* deliveriesCounted("duplicate", "direct")) - before, 75);
1507
+ const rows = (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true }))
1508
+ .deliveries;
1509
+ assert.equal(rows.filter((row) => row.outcome === "duplicate").length, 10);
1510
+ assert.equal(rows.filter((row) => row.outcome === "accepted").length, 1);
1511
+ const keys = yield* sql<{
1512
+ replay_key: string;
1513
+ }>`SELECT replay_key FROM scheduled_task_webhook_replay_keys`;
1514
+ assert.match(keys[0]!.replay_key, /^[a-f0-9]{64}$/);
1515
+ for (let i = 0; i < 59; i++) {
1516
+ const bodyText = JSON.stringify({ i });
1517
+ const result = yield* service.triggerWebhook(
1518
+ requestFor(task, { bodyText, body: new TextEncoder().encode(bodyText) }),
1519
+ );
1520
+ assert.equal(result._tag, "accepted");
1521
+ yield* Queue.take(launches);
1522
+ }
1523
+ }),
1524
+ ),
1525
);
1526
1037
-const pullRequestFilterInput = () =>
1038
- webhookTaskInput({
1039
- schedule: {
1040
- type: "webhook",
1041
- filters: [{ path: "headers.x-github-event", op: "equals", value: "pull_request" }],
1042
- },
1043
- });
1044
-const githubEvent = (event: string) => ({
1045
- headers: { "content-type": "application/json", "x-github-event": event },
1527
+it.effect("caps coalesced history while persisting the latest overflow payload", () => {
1528
+ const workers = new Map<string, Effect.Effect<void>>();
1529
+ return withService(
1530
+ ({ service, sql, launches }) =>
1531
+ Effect.gen(function* () {
1532
+ const { task } = yield* service.upsert(
1533
+ yield* webhookTaskInput({
1534
+ prompt: "{{body.value}}",
1535
+ schedule: {
1536
+ type: "webhook",
1537
+ dedupe: { replayKey: "{{headers.delivery-id}}", coalesceKey: "{{body.group}}" },
1538
+ },
1539
+ }),
1540
+ );
1541
+ const request = (i: number) => {
1542
+ const bodyText = JSON.stringify({ group: "B", value: i });
1543
+ return requestFor(task, {
1544
+ bodyText,
1545
+ body: new TextEncoder().encode(bodyText),
1546
+ headers: { "delivery-id": String(i) },
1547
+ });
1548
+ };
1549
+ for (let i = 0; i < 40; i++)
1550
+ assert.equal((yield* service.triggerWebhook(request(i)))._tag, "accepted");
1551
+ assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_replay_keys`).length, 40);
1552
+ const rows = (yield* service.listWebhookDeliveries({
1553
+ id: task.id,
1554
+ acceptAnyOutcome: true,
1555
+ })).deliveries;
1556
+ assert.equal(rows.filter((row) => row.outcome === "coalesced").length, 10);
1557
+ assert.equal(rows.filter((row) => row.outcome === "accepted").length, 1);
1558
+ const pending = yield* sql<{
1559
+ coalesce_key: string;
1560
+ }>`SELECT coalesce_key FROM scheduled_task_webhook_pending`;
1561
+ assert.match(pending[0]!.coalesce_key, /^[a-f0-9]{64}$/);
1562
+ yield* TestClock.adjust("120 seconds");
1563
+ yield* drainPendingDispatch(workers);
1564
+ assert.include((yield* Queue.take(launches)).initialMessage!.text, ">39<");
1565
+ assert.equal(yield* Queue.size(launches), 0);
1566
+ }),
1567
+ { workers },
1568
+ );
1569
});
1570
1048
-it.effect("filtered deliveries take no run slot and log at most 10 rows a minute", () =>
1571
+it.effect("a partially missing key is empty even with a literal prefix", () =>
1572
withService(({ service, launches }) =>
1573
Effect.gen(function* () {
1051
- const { task } = yield* service.upsert(yield* pullRequestFilterInput());
1052
- const ignored = yield* Effect.forEach(Array.from({ length: 70 }), () =>
1053
- service.triggerWebhook(requestFor(task, githubEvent("push"))),
1574
+ const { task } = yield* service.upsert(
1575
+ yield* webhookTaskInput({
1576
+ schedule: {
1577
+ type: "webhook",
1578
+ dedupe: { replayKey: "gh:{{headers.absent}}", coalesceKey: "pr:{{body.absent}}" },
1579
+ },
1580
+ }),
1581
);
1055
- assert.isTrue(
1582
+ for (let i = 0; i < 2; i++) {
1583
+ const result = yield* service.triggerWebhook(requestFor(task, { headers: { absent: "" } }));
1584
+ assert.equal(result._tag === "accepted" && result.outcome, "accepted");
1585
+ yield* Queue.take(launches);
1586
+ }
1587
+ }),
1588
+ ),
1589
+);
1590
+
1591
+it.effect("queue-full deliveries release replay keys so a later retry runs", () =>
1592
+ Effect.gen(function* () {
1593
+ const gate = yield* Deferred.make<void>();
1594
+ yield* withService(
1595
+ ({ service, launches }) =>
1596
+ Effect.gen(function* () {
1597
+ const { task } = yield* service.upsert(
1598
+ yield* webhookTaskInput({
1599
+ schedule: { type: "webhook", dedupe: { replayKey: "{{headers.delivery-id}}" } },
1600
+ }),
1601
+ );
1602
+ const request = (i: number) =>
1603
+ requestFor(task, { headers: { "delivery-id": String(i) } });
1604
+ for (let i = 0; i < 20; i++) yield* service.triggerWebhook(request(i));
1605
+ yield* Queue.take(launches);
1606
+ const full = yield* service.triggerWebhook(request(20));
1607
+ assert.equal(full._tag === "rate_limited" && full.outcome, "queue_full");
1608
+ yield* Deferred.succeed(gate, undefined);
1609
+ for (let i = 1; i < 20; i++) yield* Queue.take(launches);
1610
+ const retry = yield* service.triggerWebhook(request(20));
1611
+ assert.equal(retry._tag === "accepted" && retry.outcome, "accepted");
1612
+ yield* Queue.take(launches);
1613
+ }),
1614
+ { gate },
1615
+ );
1616
+ }),
1617
+);
1618
+
1619
+it.effect.each(["task_json", "request_json"] as const)(
1620
+ "removes corrupt pending %s instead of retrying forever",
1621
+ (column) => {
1622
+ const workers = new Map<string, Effect.Effect<void>>();
1623
+ return withService(
1624
+ ({ service, sql }) =>
1625
+ Effect.gen(function* () {
1626
+ const { task } = yield* service.upsert(
1627
+ yield* webhookTaskInput({
1628
+ schedule: {
1629
+ type: "webhook",
1630
+ dedupe: {
1631
+ replayKey: "{{body}}",
1632
+ coalesceKey: "{{body}}",
1633
+ coalesceWindowSeconds: 10,
1634
+ },
1635
+ },
1636
+ }),
1637
+ );
1638
+ yield* service.triggerWebhook(requestFor(task));
1639
+ yield* sql`UPDATE scheduled_task_webhook_pending SET ${sql(column)} = 'broken'`;
1640
+ yield* TestClock.adjust("10 seconds");
1641
+ yield* drainPendingDispatch(workers);
1642
+ assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_pending`).length, 0);
1643
+ assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_replay_keys`).length, 0);
1644
+ assert.equal(
1645
+ (yield* service.listWebhookDeliveries({ id: task.id })).deliveries[0]!.outcome,
1646
+ "dispatch_failed",
1647
+ );
1648
+ yield* workers.get("webhook-coalescing")!;
1649
+ }),
1650
+ { workers },
1651
+ );
1652
+ },
1653
+);
1654
+
1655
+it.effect(
1656
+ "restarting after a coalesced launch before pending deletion does not launch or count it twice",
1657
+ () => {
1658
+ const workers = new Map<string, Effect.Effect<void>>();
1659
+ return withService(
1660
+ ({ service, sql, launches }) =>
1661
+ Effect.gen(function* () {
1662
+ const { task } = yield* service.upsert(
1663
+ yield* webhookTaskInput({
1664
+ schedule: {
1665
+ type: "webhook",
1666
+ dedupe: { coalesceKey: "{{body}}", coalesceWindowSeconds: 10 },
1667
+ },
1668
+ }),
1669
+ );
1670
+ const accepted = yield* service.triggerWebhook(requestFor(task));
1671
+ yield* sql`CREATE TRIGGER fail_pending_delete BEFORE DELETE ON scheduled_task_webhook_pending BEGIN SELECT RAISE(FAIL,'crash after launch'); END`;
1672
+ yield* TestClock.adjust("10 seconds");
1673
+ yield* drainPendingDispatch(workers);
1674
+ const launch = yield* Queue.take(launches);
1675
+ assert.equal(
1676
+ launch.commandId,
1677
+ `scheduled-task:${task.id}:webhook:${accepted._tag === "accepted" && accepted.deliveryId}`,
1678
+ );
1679
+ assert.equal((yield* service.list()).tasks[0]!.runCount, 1);
1680
+ assert.equal(
1681
+ (yield* sql`SELECT * FROM scheduled_task_webhook_pending WHERE state = 'completed'`)
1682
+ .length,
1683
+ 1,
1684
+ );
1685
+ yield* sql`DROP TRIGGER fail_pending_delete`;
1686
+ const restartedWorkers = new Map<string, Effect.Effect<void>>();
1687
+ yield* withService(
1688
+ ({ service: restarted, launches: secondLaunches }) =>
1689
+ Effect.gen(function* () {
1690
+ yield* drainPendingDispatch(restartedWorkers);
1691
+ assert.equal(yield* Queue.size(secondLaunches), 0);
1692
+ assert.equal((yield* restarted.list()).tasks[0]!.runCount, 1);
1693
+ assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_pending`).length, 0);
1694
+ }),
1695
+ { sql, workers: restartedWorkers },
1696
+ );
1697
+ }),
1698
+ { workers },
1699
+ );
1700
+ },
1701
+);
1702
+
1703
+it.effect("many windows on a busy task share one worker without delaying another task", () =>
1704
+ Effect.gen(function* () {
1705
+ const gate = yield* Deferred.make<void>();
1706
+ const workers = new Map<string, Effect.Effect<void>>();
1707
+ yield* withService(
1708
+ ({ service, launches, sql }) =>
1709
+ Effect.gen(function* () {
1710
+ const schedule = {
1711
+ type: "webhook",
1712
+ dedupe: { coalesceKey: "{{body.group}}", coalesceWindowSeconds: 10 },
1713
+ };
1714
+ const { task: a } = yield* service.upsert(
1715
+ yield* webhookTaskInput({ id: "scheduled-task:a", title: "A", schedule }),
1716
+ );
1717
+ const { task: b } = yield* service.upsert(
1718
+ yield* webhookTaskInput({ id: "scheduled-task:b", title: "B", schedule }),
1719
+ );
1720
+ yield* service.triggerWebhook(requestFor(a));
1721
+ assert.equal((yield* Queue.take(launches)).title, "A");
1722
+ const pendingRequest = (task: typeof a, group: string) => {
1723
+ const bodyText = JSON.stringify({ group });
1724
+ return requestFor(task, { bodyText, body: new TextEncoder().encode(bodyText) });
1725
+ };
1726
+ for (let i = 0; i < 19; i++) {
1727
+ const result = yield* service.triggerWebhook(pendingRequest(a, String(i)));
1728
+ assert.equal(result._tag === "accepted" && result.outcome, "accepted");
1729
+ }
1730
+ const overflow = pendingRequest(a, "overflow");
1731
+ const full = yield* service.triggerWebhook({ ...overflow, relayDeliveryId: "overflow" });
1732
+ assert.equal(full._tag === "rate_limited" && full.outcome, "queue_full");
1733
+ assert.equal((yield* service.triggerWebhook(requestFor(a)))._tag, "rate_limited");
1734
+ yield* service.triggerWebhook(pendingRequest(b, "B"));
1735
+ let dispatchWorkers = 0;
1736
+ const drained = yield* Deferred.make<void>();
1737
+ let finishedWorkers = 0;
1738
+ const tracer = Tracer.make({
1739
+ span: (options) => {
1740
+ if (options.name === "ScheduledTaskService.dispatchPendingWebhook") dispatchWorkers++;
1741
+ const span = new Tracer.NativeSpan(options);
1742
+ const end = span.end.bind(span);
1743
+ span.end = (...args) => {
1744
+ end(...args);
1745
+ if (
1746
+ options.name === "ScheduledTaskService.dispatchPendingWebhook" &&
1747
+ ++finishedWorkers === 2
1748
+ )
1749
+ Deferred.doneUnsafe(drained, Effect.void);
1750
+ };
1751
+ return span;
1752
+ },
1753
+ });
1754
+ yield* TestClock.adjust("10 seconds");
1755
+ yield* workers.get("webhook-coalescing")!.pipe(Effect.withTracer(tracer));
1756
+ assert.equal((yield* Queue.take(launches)).title, "B");
1757
+ for (let i = 0; i < 10; i++)
1758
+ yield* workers.get("webhook-coalescing")!.pipe(Effect.withTracer(tracer));
1759
+ assert.equal(dispatchWorkers, 2);
1760
+ assert.equal(
1761
+ (yield* sql`SELECT 1 FROM scheduled_task_webhook_pending WHERE task_id = ${a.id}`)
1762
+ .length,
1763
+ 19,
1764
+ );
1765
+ yield* Deferred.succeed(gate, undefined);
1766
+ yield* Deferred.await(drained);
1767
+ for (let i = 0; i < 19; i++) assert.equal((yield* Queue.take(launches)).title, "A");
1768
+ const retried = yield* service.triggerWebhook({
1769
+ ...overflow,
1770
+ relayDeliveryId: "overflow",
1771
+ });
1772
+ assert.equal(retried._tag === "accepted" && retried.outcome, "accepted");
1773
+ yield* TestClock.adjust("10 seconds");
1774
+ yield* drainPendingDispatch(workers);
1775
+ assert.equal((yield* Queue.take(launches)).title, "A");
1776
+ }),
1777
+ { workers, gateForTask: (input) => (input.title === "A" ? gate : undefined) },
1778
+ );
1779
+ }),
1780
+);
1781
+
1782
+it.effect("guards bad-signature floods separately from valid run admission", () =>
1783
+ withService(({ service, sql }) =>
1784
+ Effect.gen(function* () {
1785
+ const { task } = yield* service.upsert(
1786
+ yield* webhookTaskInput({
1787
+ schedule: {
1788
+ type: "webhook",
1789
+ signature: { header: "signature", encoding: "hex", prefix: "", secret: "secret" },
1790
+ },
1791
+ }),
1792
+ );
1793
+ const results = yield* Effect.forEach(Array.from({ length: 75 }), () =>
1794
+ service.triggerWebhook(requestFor(task)),
1795
+ );
1796
+ assert.equal(results.filter((result) => result._tag === "rejected_signature").length, 60);
1797
+ assert.equal(results.filter((result) => result._tag === "rate_limited").length, 15);
1798
+ assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_deliveries`).length, 51);
1799
+ }),
1800
+ ),
1801
+);
1802
+
1803
+it.effect("start failures release both replay keys and relay claims for redelivery", () =>
1804
+ withService(
1805
+ ({ service, sql, launches }) =>
1806
+ Effect.gen(function* () {
1807
+ const { task } = yield* service.upsert(
1808
+ yield* webhookTaskInput({
1809
+ schedule: { type: "webhook", dedupe: { replayKey: "{{body}}" } },
1810
+ }),
1811
+ );
1812
+ const request = requestFor(task, { relayDeliveryId: "failed-start" });
1813
+ yield* waitForSpan(
1814
+ service.triggerWebhook(request),
1815
+ "ScheduledTaskService.runWebhookDelivery",
1816
+ );
1817
+ yield* Queue.take(launches);
1818
+ assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_replay_keys`).length, 0);
1819
+ assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_relay_deliveries`).length, 0);
1820
+ const retry = yield* waitForSpan(
1821
+ service.triggerWebhook(request),
1822
+ "ScheduledTaskService.runWebhookDelivery",
1823
+ );
1824
+ assert.equal(retry._tag === "accepted" && retry.outcome, "accepted");
1825
+ yield* Queue.take(launches);
1826
+ assert.equal((yield* service.list()).tasks[0]!.lastRunStatus, "succeeded");
1827
+ }),
1828
+ { failLaunches: { remaining: 1 } },
1829
+ ),
1830
+);
1831
+
1832
+it.effect("bounds oversized prompt floods without evicting run or filtered history", () =>
1833
+ withService(({ service, sql, launches }) =>
1834
+ Effect.gen(function* () {
1835
+ const { task } = yield* service.upsert(yield* webhookTaskInput({ prompt: "{{body}}" }));
1836
+ const accepted = yield* service.triggerWebhook(requestFor(task));
1837
+ yield* Queue.take(launches);
1838
+ assert.equal(accepted._tag, "accepted");
1839
+ // A sibling filter step owns these rows; suppression must preserve its bucket too.
1840
+ yield* sql`INSERT INTO scheduled_task_webhook_deliveries
1841
+ SELECT 'filtered-fixture', task_id, received_at, method, query, headers_json, body,
1842
+ body_bytes, body_truncated, 'filtered', signature_verified, missing_fields_json, rendered_prompt, error
1843
+ FROM scheduled_task_webhook_deliveries LIMIT 1`;
1844
+ const bodyText = "x".repeat(200_000);
1845
+ const request = requestFor(task, { bodyText, body: new TextEncoder().encode(bodyText) });
1846
+ const outcomes = yield* Effect.forEach(Array.from({ length: 75 }), () =>
1847
+ service.triggerWebhook(request),
1848
+ );
1849
+ assert.equal(outcomes.filter((result) => result._tag === "accepted").length, 60);
1850
+ assert.equal(outcomes.filter((result) => result._tag === "rate_limited").length, 15);
1851
+ const rows = yield* sql<{
1852
+ outcome: string;
1853
+ }>`SELECT outcome FROM scheduled_task_webhook_deliveries`;
1854
+ assert.equal(rows.filter((row) => row.outcome === "accepted").length, 1);
1855
+ assert.equal(rows.filter((row) => row.outcome === "filtered").length, 1);
1856
+ assert.equal(rows.length, 22);
1857
+ assert.equal(yield* Queue.size(launches), 0);
1858
+ yield* TestClock.adjust("61 seconds");
1859
+ assert.equal((yield* service.triggerWebhook(requestFor(task)))._tag, "accepted");
1860
+ yield* Queue.take(launches);
1861
+ }),
1862
+ ),
1863
+);
1864
+
1865
+it.effect("single delivery lookup hides new outcomes unless the client opts in", () =>
1866
+ withService(({ service, launches }) =>
1867
+ Effect.gen(function* () {
1868
+ const { task } = yield* service.upsert(
1869
+ yield* webhookTaskInput({
1870
+ schedule: {
1871
+ type: "webhook",
1872
+ dedupe: { replayKey: "{{body}}", coalesceKey: "{{body.group}}" },
1873
+ },
1874
+ }),
1875
+ );
1876
+ yield* service.triggerWebhook(requestFor(task));
1877
+ yield* Queue.take(launches);
1878
+ yield* service.triggerWebhook(requestFor(task));
1879
+ for (const value of [1, 2]) {
1880
+ const bodyText = JSON.stringify({ group: "B", value });
1881
+ yield* service.triggerWebhook(
1882
+ requestFor(task, { bodyText, body: new TextEncoder().encode(bodyText) }),
1883
+ );
1884
+ }
1885
+ const rows = (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true }))
1886
+ .deliveries;
1887
+ for (const row of rows.filter(
1888
+ (row) => row.outcome === "duplicate" || row.outcome === "coalesced",
1889
+ )) {
1890
+ const input = { id: task.id, deliveryId: row.id };
1891
+ const hidden = yield* service.getWebhookDelivery(input).pipe(Effect.result);
1892
+ assert.equal(hidden._tag, "Failure");
1893
+ assert.equal(
1894
+ (yield* service.getWebhookDelivery({ ...input, acceptAnyOutcome: true })).delivery
1895
+ .outcome,
1896
+ row.outcome,
1897
+ );
1898
+ }
1899
+ assert.equal(
1900
+ rows.filter((row) => row.outcome === "duplicate" || row.outcome === "coalesced").length,
1901
+ 2,
1902
+ );
1903
+ }),
1904
+ ),
1905
+);
1906
+
1907
+it.effect("main retention keeps fifty runs and preserves filtered and dedupe buckets", () =>
1908
+ withService(({ service, sql, launches }) =>
1909
+ Effect.gen(function* () {
1910
+ const { task } = yield* service.upsert(yield* webhookTaskInput());
1911
+ yield* service.triggerWebhook(requestFor(task));
1912
+ yield* Queue.take(launches);
1913
+ for (const outcome of ["filtered", "duplicate"]) {
1914
+ yield* sql`WITH RECURSIVE n(value) AS (SELECT 1 UNION ALL SELECT value + 1 FROM n WHERE value < 20)
1915
+ INSERT INTO scheduled_task_webhook_deliveries
1916
+ SELECT ${outcome} || n.value, d.task_id, d.received_at, d.method, d.query, d.headers_json, d.body,
1917
+ d.body_bytes, d.body_truncated, ${outcome}, d.signature_verified, d.missing_fields_json, d.rendered_prompt, d.error
1918
+ FROM n CROSS JOIN (SELECT * FROM scheduled_task_webhook_deliveries WHERE outcome = 'accepted' LIMIT 1) d`;
1919
+ }
1920
+ for (let i = 0; i < 59; i++) {
1921
+ assert.equal((yield* service.triggerWebhook(requestFor(task)))._tag, "accepted");
1922
+ yield* Queue.take(launches);
1923
+ }
1924
+ const counts = yield* sql<{
1925
+ outcome: string;
1926
+ count: number;
1927
+ }>`SELECT outcome, COUNT(*) AS count FROM scheduled_task_webhook_deliveries GROUP BY outcome ORDER BY outcome`;
1928
+ assert.deepEqual(counts, [
1929
+ { outcome: "accepted", count: 50 },
1930
+ { outcome: "duplicate", count: 20 },
1931
+ { outcome: "filtered", count: 20 },
1932
+ ]);
1933
+ }),
1934
+ ),
1935
+);
1936
+
1937
+it.effect(
1938
+ "a valid request refunds its own reservation without expiring an interleaved rejection",
1939
+ () =>
1940
+ Effect.gen(function* () {
1941
+ const started = yield* Deferred.make<void>();
1942
+ const release = yield* Deferred.make<void>();
1943
+ yield* withService(
1944
+ ({ service, launches }) =>
1945
+ Effect.gen(function* () {
1946
+ const { task } = yield* service.upsert(
1947
+ yield* webhookTaskInput({
1948
+ schedule: {
1949
+ type: "webhook",
1950
+ signature: { header: "signature", encoding: "hex", prefix: "", secret: "secret" },
1951
+ },
1952
+ }),
1953
+ );
1954
+ const signature = NodeCrypto.createHmac("sha256", "secret")
1955
+ .update(pullRequestBody)
1956
+ .digest("hex");
1957
+ const valid = yield* service
1958
+ .triggerWebhook(requestFor(task, { headers: { signature } }))
1959
+ .pipe(Effect.forkChild);
1960
+ yield* Deferred.await(started);
1961
+ yield* TestClock.adjust("30 seconds");
1962
+ assert.equal(
1963
+ (yield* service.triggerWebhook(requestFor(task)))._tag,
1964
+ "rejected_signature",
1965
+ );
1966
+ yield* Deferred.succeed(release, undefined);
1967
+ assert.equal((yield* Fiber.join(valid))._tag, "accepted");
1968
+ yield* Queue.take(launches);
1969
+ // Only the valid reservation would have expired now; the rejection must still count.
1970
+ yield* TestClock.adjust("31 seconds");
1971
+ for (let i = 0; i < 59; i++)
1972
+ assert.equal(
1973
+ (yield* service.triggerWebhook(requestFor(task)))._tag,
1974
+ "rejected_signature",
1975
+ );
1976
+ assert.equal((yield* service.triggerWebhook(requestFor(task)))._tag, "rate_limited");
1977
+ }),
1978
+ { frameGate: { started, release } },
1979
+ );
1980
+ }),
1981
+);
1982
+
1983
+it.effect("keeps replay keys when dispatch succeeds but completion bookkeeping fails", () =>
1984
+ withService(({ service, sql, launches }) =>
1985
+ Effect.gen(function* () {
1986
+ const { task } = yield* service.upsert(
1987
+ yield* webhookTaskInput({
1988
+ schedule: {
1989
+ type: "webhook",
1990
+ dedupe: { replayKey: "{{body}}" },
1991
+ },
1992
+ }),
1993
+ );
1994
+ yield* sql`CREATE TRIGGER fail_completion BEFORE UPDATE ON scheduled_tasks
1995
+ WHEN NEW.last_run_status = 'succeeded' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`;
1996
+ yield* waitForSpan(
1997
+ service.triggerWebhook(requestFor(task)),
1998
+ "ScheduledTaskService.runWebhookDelivery",
1999
+ );
2000
+ yield* Queue.take(launches);
2001
+ assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`).length, 1);
2002
+ yield* sql`DROP TRIGGER fail_completion`;
2003
+ const retry = yield* service.triggerWebhook(requestFor(task));
2004
+ assert.equal(retry._tag === "accepted" && retry.outcome, "duplicate");
2005
+ assert.equal(yield* Queue.size(launches), 0);
2006
+ assert.equal((yield* service.list()).tasks[0]!.runCount, 1);
2007
+ }),
2008
+ ),
2009
+);
2010
+
2011
+it.effect("restart before a coalesced completion commits counts the delivery once", () =>
2012
+ Effect.gen(function* () {
2013
+ const gate = yield* Deferred.make<void>();
2014
+ const completedLaunches = new Set<string>();
2015
+ const workers = new Map<string, Effect.Effect<void>>();
2016
+ yield* withService(
2017
+ ({ service, sql, launches }) =>
2018
+ Effect.gen(function* () {
2019
+ const { task } = yield* service.upsert(
2020
+ yield* webhookTaskInput({
2021
+ schedule: {
2022
+ type: "webhook",
2023
+ dedupe: {
2024
+ replayKey: "{{body}}",
2025
+ coalesceKey: "{{body}}",
2026
+ coalesceWindowSeconds: 10,
2027
+ },
2028
+ },
2029
+ }),
2030
+ );
2031
+ yield* service.triggerWebhook(requestFor(task));
2032
+ yield* TestClock.adjust("10 seconds");
2033
+ yield* workers.get("webhook-coalescing")!;
2034
+ const launch = yield* Queue.take(launches);
2035
+ assert.isTrue(completedLaunches.has(launch.commandId));
2036
+ assert.equal(
2037
+ (yield* sql<{ state: string }>`SELECT state FROM scheduled_task_webhook_pending`)[0]!
2038
+ .state,
2039
+ "running",
2040
+ );
2041
+ assert.equal(
2042
+ (yield* sql`SELECT 1 FROM scheduled_task_webhook_pending WHERE state = 'completed'`)
2043
+ .length,
2044
+ 0,
2045
+ );
2046
+ assert.equal((yield* service.list()).tasks[0]!.lastRunStatus, "running");
2047
+ const restartedWorkers = new Map<string, Effect.Effect<void>>();
2048
+ yield* withService(
2049
+ ({ service: restarted, launches: recoveredLaunches }) =>
2050
+ Effect.gen(function* () {
2051
+ assert.equal((yield* restarted.list()).tasks[0]!.runCount, 0);
2052
+ yield* drainPendingDispatch(restartedWorkers);
2053
+ assert.equal(yield* Queue.size(recoveredLaunches), 0);
2054
+ assert.equal((yield* restarted.list()).tasks[0]!.runCount, 1);
2055
+ assert.equal((yield* restarted.list()).tasks[0]!.lastRunStatus, "succeeded");
2056
+ assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0);
2057
+ const retry = yield* restarted.triggerWebhook(requestFor(task));
2058
+ assert.equal(retry._tag === "accepted" && retry.outcome, "duplicate");
2059
+ }),
2060
+ { sql, workers: restartedWorkers, completedLaunches },
2061
+ );
2062
+ }),
2063
+ { workers, gate, completedLaunches },
2064
+ );
2065
+ }),
2066
+);
2067
+
2068
+it.effect.each([119_000, 120_000])(
2069
+ "bounds coalesced prompt logs and checks trimmed length for %i body characters",
2070
+ (length) => {
2071
+ const workers = new Map<string, Effect.Effect<void>>();
2072
+ return withService(
2073
+ ({ service, launches }) =>
2074
+ Effect.gen(function* () {
2075
+ const { task } = yield* service.upsert(
2076
+ yield* webhookTaskInput({
2077
+ prompt: `{{body.absent}}${" ".repeat(10_000)}{{body.text}}`,
2078
+ schedule: {
2079
+ type: "webhook",
2080
+ dedupe: { coalesceKey: "{{body.group}}", coalesceWindowSeconds: 10 },
2081
+ },
2082
+ }),
2083
+ );
2084
+ const bodyText = JSON.stringify({ group: "B", text: "x".repeat(length) });
2085
+ const accepted = yield* service.triggerWebhook(
2086
+ requestFor(task, { bodyText, body: new TextEncoder().encode(bodyText) }),
2087
+ );
2088
+ assert.equal(accepted._tag, "accepted");
2089
+ yield* TestClock.adjust("10 seconds");
2090
+ yield* drainPendingDispatch(workers);
2091
+ const delivery = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries[0]!;
2092
+ if (length === 119_000) {
2093
+ const launch = yield* Queue.take(launches);
2094
+ assert.isAbove(launch.initialMessage!.text.length, 120_000);
2095
+ assert.equal(delivery.outcome, "accepted");
2096
+ const detail = yield* service.getWebhookDelivery({
2097
+ id: task.id,
2098
+ deliveryId: delivery.id,
2099
+ });
2100
+ assert.equal(detail.delivery.renderedPrompt!.length, 64 * 1024);
2101
+ } else {
2102
+ assert.equal(yield* Queue.size(launches), 0);
2103
+ assert.equal(delivery.outcome, "dispatch_failed");
2104
+ assert.equal(delivery.error, "The filled-in prompt is too long.");
2105
+ }
2106
+ }),
2107
+ { workers },
2108
+ );
2109
+ },
2110
+);
2111
+
2112
+it.effect("retries coalesced completion bookkeeping without launching or counting twice", () => {
2113
+ const workers = new Map<string, Effect.Effect<void>>();
2114
+ return withService(
2115
+ ({ service, sql, launches }) =>
2116
+ Effect.gen(function* () {
2117
+ const { task } = yield* service.upsert(
2118
+ yield* webhookTaskInput({
2119
+ schedule: {
2120
+ type: "webhook",
2121
+ dedupe: { replayKey: "{{body}}", coalesceKey: "{{body}}", coalesceWindowSeconds: 10 },
2122
+ },
2123
+ }),
2124
+ );
2125
+ yield* service.triggerWebhook(requestFor(task));
2126
+ yield* sql`CREATE TRIGGER fail_coalesced_completion BEFORE UPDATE ON scheduled_tasks
2127
+ WHEN NEW.last_run_status = 'succeeded' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`;
2128
+ yield* TestClock.adjust("10 seconds");
2129
+ yield* drainPendingDispatch(workers);
2130
+ yield* Queue.take(launches);
2131
+ assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 1);
2132
+ assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`).length, 1);
2133
+ assert.equal(
2134
+ (yield* sql`SELECT 1 FROM scheduled_task_webhook_pending WHERE state = 'completed'`)
2135
+ .length,
2136
+ 0,
2137
+ );
2138
+ assert.equal((yield* service.list()).tasks[0]!.runCount, 0);
2139
+ yield* sql`DROP TRIGGER fail_coalesced_completion`;
2140
+ yield* drainPendingDispatch(workers);
2141
+ assert.equal(yield* Queue.size(launches), 0);
2142
+ assert.equal((yield* service.list()).tasks[0]!.runCount, 1);
2143
+ assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0);
2144
+ const delivery = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries[0]!;
2145
+ assert.equal(delivery.outcome, "accepted");
2146
+ assert.equal(delivery.error, null);
2147
+ }),
2148
+ { workers, completedLaunches: new Set() },
2149
+ );
2150
+});
2151
+
2152
+it.effect("propagates coalescing log failures with their task id and rolls back acceptance", () => {
2153
+ const workers = new Map<string, Effect.Effect<void>>();
2154
+ return withService(
2155
+ ({ service, sql, launches }) =>
2156
+ Effect.gen(function* () {
2157
+ const { task } = yield* service.upsert(
2158
+ yield* webhookTaskInput({
2159
+ schedule: {
2160
+ type: "webhook",
2161
+ dedupe: { replayKey: "{{body}}", coalesceKey: "{{body}}", coalesceWindowSeconds: 10 },
2162
+ },
2163
+ }),
2164
+ );
2165
+ const request = requestFor(task, { relayDeliveryId: "coalescing-log-failure" });
2166
+ yield* sql`CREATE TRIGGER fail_coalescing_log BEFORE INSERT ON scheduled_task_webhook_deliveries
2167
+ BEGIN SELECT RAISE(FAIL, 'injected coalescing log failure'); END`;
2168
+ const error = yield* service.triggerWebhook(request).pipe(Effect.flip);
2169
+ assert.equal(error.message, "Could not record webhook delivery.");
2170
+ assert.equal(error.taskId, task.id);
2171
+ assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0);
2172
+ assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`).length, 0);
2173
+ assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_relay_deliveries`).length, 0);
2174
+ yield* sql`DROP TRIGGER fail_coalescing_log`;
2175
+ assert.equal((yield* service.triggerWebhook(request))._tag, "accepted");
2176
+ yield* TestClock.adjust("10 seconds");
2177
+ yield* drainPendingDispatch(workers);
2178
+ yield* Queue.take(launches);
2179
+ }),
2180
+ { workers },
2181
+ );
2182
+});
2183
+
2184
+it.effect.each([1, 11])(
2185
+ "keeps relay claims for %i coalesced updates after their window closes",
2186
+ (count) => {
2187
+ const workers = new Map<string, Effect.Effect<void>>();
2188
+ return withService(
2189
+ ({ service, sql, launches }) =>
2190
+ Effect.gen(function* () {
2191
+ const { task } = yield* service.upsert(
2192
+ yield* webhookTaskInput({
2193
+ prompt: "{{body.value}}",
2194
+ schedule: {
2195
+ type: "webhook",
2196
+ dedupe: { coalesceKey: "{{body.group}}", coalesceWindowSeconds: 10 },
2197
+ },
2198
+ }),
2199
+ );
2200
+ const request = (value: number) => {
2201
+ const bodyText = JSON.stringify({ group: "B", value });
2202
+ return requestFor(task, {
2203
+ relayDeliveryId: `coalesced-relay-${value}`,
2204
+ bodyText,
2205
+ body: new TextEncoder().encode(bodyText),
2206
+ });
2207
+ };
2208
+ yield* service.triggerWebhook(request(0));
2209
+ for (let i = 1; i <= count; i++) {
2210
+ const result = yield* service.triggerWebhook(request(i));
2211
+ assert.equal(result._tag === "accepted" && result.outcome, "coalesced");
2212
+ }
2213
+ assert.equal(
2214
+ (yield* sql`SELECT 1 FROM scheduled_task_webhook_relay_deliveries`).length,
2215
+ count + 1,
2216
+ );
2217
+ yield* TestClock.adjust("10 seconds");
2218
+ yield* drainPendingDispatch(workers);
2219
+ assert.include((yield* Queue.take(launches)).initialMessage!.text, `>${count}<`);
2220
+ const retry = yield* service.triggerWebhook(request(count));
2221
+ assert.equal(retry._tag === "accepted" && retry.outcome, "duplicate");
2222
+ assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0);
2223
+ assert.equal((yield* service.list()).tasks[0]!.runCount, 1);
2224
+ assert.equal(yield* Queue.size(launches), 0);
2225
+ }),
2226
+ { workers },
2227
+ );
2228
+ },
2229
+);
2230
+
2231
+it.effect("failed windows release every member's replay key and relay claim", () => {
2232
+ const workers = new Map<string, Effect.Effect<void>>();
2233
+ return withService(
2234
+ ({ service, sql, launches }) =>
2235
+ Effect.gen(function* () {
2236
+ const { task } = yield* service.upsert(
2237
+ yield* webhookTaskInput({
2238
+ prompt: "{{body.value}}",
2239
+ schedule: {
2240
+ type: "webhook",
2241
+ dedupe: {
2242
+ replayKey: "{{headers.delivery-id}}",
2243
+ coalesceKey: "{{body.group}}",
2244
+ coalesceWindowSeconds: 10,
2245
+ },
2246
+ },
2247
+ }),
2248
+ );
2249
+ const request = (id: string) => {
2250
+ const bodyText = JSON.stringify({ group: "same", value: id });
2251
+ return requestFor(task, {
2252
+ bodyText,
2253
+ body: new TextEncoder().encode(bodyText),
2254
+ headers: { "delivery-id": id },
2255
+ relayDeliveryId: id,
2256
+ });
2257
+ };
2258
+ yield* service.triggerWebhook(request("A"));
2259
+ for (let i = 0; i < 12; i++) yield* service.triggerWebhook(request(`B${i}`));
2260
+ assert.equal(
2261
+ (yield* sql`SELECT 1 FROM scheduled_task_webhook_relay_deliveries`).length,
2262
+ 13,
2263
+ );
2264
+ yield* TestClock.adjust("10 seconds");
2265
+ yield* drainPendingDispatch(workers);
2266
+ yield* Queue.take(launches);
2267
+ assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`).length, 0);
2268
+ assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_relay_deliveries`).length, 0);
2269
+ const retry = yield* service.triggerWebhook(request("B11"));
2270
+ assert.equal(retry._tag === "accepted" && retry.outcome, "accepted");
2271
+ yield* TestClock.adjust("10 seconds");
2272
+ yield* drainPendingDispatch(workers);
2273
+ assert.include((yield* Queue.take(launches)).initialMessage!.text, "B11");
2274
+ assert.equal((yield* service.list()).tasks[0]!.runCount, 2);
2275
+ }),
2276
+ { workers, failLaunches: { remaining: 1 } },
2277
+ );
2278
+});
2279
+
2280
+it.effect(
2281
+ "rejects a twenty-first persisted window and accepts its relay retry after draining",
2282
+ () => {
2283
+ const workers = new Map<string, Effect.Effect<void>>();
2284
+ return withService(
2285
+ ({ service, sql, launches }) =>
2286
+ Effect.gen(function* () {
2287
+ const { task } = yield* service.upsert(
2288
+ yield* webhookTaskInput({
2289
+ prompt: "{{body.group}}",
2290
+ schedule: {
2291
+ type: "webhook",
2292
+ dedupe: {
2293
+ replayKey: "{{body.group}}",
2294
+ coalesceKey: "{{body.group}}",
2295
+ coalesceWindowSeconds: 10,
2296
+ },
2297
+ },
2298
+ }),
2299
+ );
2300
+ const request = (group: number) => {
2301
+ const bodyText = JSON.stringify({ group });
2302
+ return requestFor(task, {
2303
+ bodyText,
2304
+ body: new TextEncoder().encode(bodyText),
2305
+ relayDeliveryId: `window-${group}`,
2306
+ });
2307
+ };
2308
+ for (let i = 0; i < 20; i++)
2309
+ assert.equal((yield* service.triggerWebhook(request(i)))._tag, "accepted");
2310
+ const overflow = yield* service.triggerWebhook(request(20));
2311
+ assert.equal(overflow._tag === "rate_limited" && overflow.outcome, "queue_full");
2312
+ assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 20);
2313
+ assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`).length, 20);
2314
+ assert.equal(
2315
+ (yield* sql`SELECT 1 FROM scheduled_task_webhook_relay_deliveries`).length,
2316
+ 20,
2317
+ );
2318
+ yield* TestClock.adjust("10 seconds");
2319
+ yield* drainPendingDispatch(workers);
2320
+ for (let i = 0; i < 20; i++) yield* Queue.take(launches);
2321
+ assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0);
2322
+ const retry = yield* service.triggerWebhook(request(20));
2323
+ assert.equal(retry._tag === "accepted" && retry.outcome, "accepted");
2324
+ yield* TestClock.adjust("10 seconds");
2325
+ yield* drainPendingDispatch(workers);
2326
+ assert.include((yield* Queue.take(launches)).initialMessage!.text, "20");
2327
+ assert.equal((yield* service.list()).tasks[0]!.runCount, 21);
2328
+ }),
2329
+ { workers },
2330
+ );
2331
+ },
2332
+);
2333
+
2334
+it.effect(
2335
+ "recovery finishes failed-window claim cleanup without counting the attempt twice",
2336
+ () => {
2337
+ const workers = new Map<string, Effect.Effect<void>>();
2338
+ return withService(
2339
+ ({ service, sql, launches }) =>
2340
+ Effect.gen(function* () {
2341
+ const { task } = yield* service.upsert(
2342
+ yield* webhookTaskInput({
2343
+ schedule: {
2344
+ type: "webhook",
2345
+ dedupe: {
2346
+ replayKey: "{{headers.delivery-id}}",
2347
+ coalesceKey: "{{body}}",
2348
+ coalesceWindowSeconds: 10,
2349
+ },
2350
+ },
2351
+ }),
2352
+ );
2353
+ const request = (id: string) =>
2354
+ requestFor(task, { headers: { "delivery-id": id }, relayDeliveryId: id });
2355
+ yield* service.triggerWebhook(request("A"));
2356
+ yield* service.triggerWebhook(request("B"));
2357
+ yield* sql`CREATE TRIGGER fail_claim_cleanup BEFORE DELETE ON scheduled_task_webhook_relay_deliveries
2358
+ BEGIN SELECT RAISE(FAIL, 'cleanup unavailable'); END`;
2359
+ yield* TestClock.adjust("10 seconds");
2360
+ yield* drainPendingDispatch(workers);
2361
+ yield* Queue.take(launches);
2362
+ assert.equal((yield* service.list()).tasks[0]!.runCount, 1);
2363
+ assert.equal(
2364
+ (yield* sql<{ state: string }>`SELECT state FROM scheduled_task_webhook_pending`)[0]!
2365
+ .state,
2366
+ "failed",
2367
+ );
2368
+ yield* sql`DROP TRIGGER fail_claim_cleanup`;
2369
+ const restartedWorkers = new Map<string, Effect.Effect<void>>();
2370
+ yield* withService(
2371
+ ({ service: restarted, launches: restartedLaunches }) =>
2372
+ Effect.gen(function* () {
2373
+ yield* drainPendingDispatch(restartedWorkers);
2374
+ assert.equal(yield* Queue.size(restartedLaunches), 0);
2375
+ assert.equal((yield* restarted.list()).tasks[0]!.runCount, 1);
2376
+ assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0);
2377
+ assert.equal(
2378
+ (yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`).length,
2379
+ 0,
2380
+ );
2381
+ assert.equal(
2382
+ (yield* sql`SELECT 1 FROM scheduled_task_webhook_relay_deliveries`).length,
2383
+ 0,
2384
+ );
2385
+ const retry = yield* restarted.triggerWebhook(request("B"));
2386
+ assert.equal(retry._tag === "accepted" && retry.outcome, "accepted");
2387
+ yield* TestClock.adjust("10 seconds");
2388
+ yield* drainPendingDispatch(restartedWorkers);
2389
+ yield* Queue.take(restartedLaunches);
2390
+ assert.equal((yield* restarted.list()).tasks[0]!.runCount, 2);
2391
+ }),
2392
+ { sql, workers: restartedWorkers },
2393
+ );
2394
+ }),
2395
+ { workers, failLaunches: { remaining: 1 } },
2396
+ );
2397
+ },
2398
+);
2399
+
2400
+it.effect("a full run budget still accepts replay hits and the latest open-window payload", () => {
2401
+ const workers = new Map<string, Effect.Effect<void>>();
2402
+ return withService(
2403
+ ({ service, launches, sql }) =>
2404
+ Effect.gen(function* () {
2405
+ const { task } = yield* service.upsert(
2406
+ yield* webhookTaskInput({
2407
+ prompt: "{{body.value}}",
2408
+ schedule: {
2409
+ type: "webhook",
2410
+ dedupe: {
2411
+ replayKey: "{{headers.delivery-id}}",
2412
+ coalesceKey: "{{body.group}}",
2413
+ coalesceWindowSeconds: 10,
2414
+ },
2415
+ },
2416
+ }),
2417
+ );
2418
+ const request = (id: string, value: string, group?: string) => {
2419
+ const bodyText = JSON.stringify({ value, ...(group === undefined ? {} : { group }) });
2420
+ return requestFor(task, {
2421
+ bodyText,
2422
+ body: new TextEncoder().encode(bodyText),
2423
+ headers: { "delivery-id": id },
2424
+ });
2425
+ };
2426
+ for (let i = 0; i < 59; i++) {
2427
+ const result = yield* waitForSpan(
2428
+ service.triggerWebhook(request(String(i), String(i))),
2429
+ "ScheduledTaskService.runWebhookDelivery",
2430
+ );
2431
+ assert.equal(result._tag === "accepted" && result.outcome, "accepted");
2432
+ yield* Queue.take(launches);
2433
+ }
2434
+ const opened = yield* service.triggerWebhook(request("B1", "old", "B"));
2435
+ assert.equal(opened._tag === "accepted" && opened.outcome, "accepted");
2436
+ const duplicate = yield* service.triggerWebhook(request("0", "duplicate"));
2437
+ assert.equal(duplicate._tag === "accepted" && duplicate.outcome, "duplicate");
2438
+ const update = yield* service.triggerWebhook(request("B2", "latest", "B"));
2439
+ assert.equal(update._tag === "accepted" && update.outcome, "coalesced");
2440
+ assert.equal(
2441
+ (yield* service.triggerWebhook(request("C", "new window", "C")))._tag,
2442
+ "rate_limited",
2443
+ );
2444
+ assert.equal(
2445
+ (yield* service.triggerWebhook(request("D", "new immediate")))._tag,
2446
+ "rate_limited",
2447
+ );
2448
+ assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 1);
2449
+ assert.equal(yield* Queue.size(launches), 0);
2450
+ yield* TestClock.adjust("10 seconds");
2451
+ yield* drainPendingDispatch(workers);
2452
+ assert.include((yield* Queue.take(launches)).initialMessage!.text, "latest");
2453
+ assert.equal(yield* Queue.size(launches), 0);
2454
+ assert.equal((yield* service.list()).tasks[0]!.runCount, 60);
2455
+ assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0);
2456
+ }),
2457
+ { workers },
2458
+ );
2459
+});
2460
+
2461
+it.effect("dispatches an overdue window exactly once after its deadline passed while stopped", () =>
2462
+ Effect.gen(function* () {
2463
+ const sql = yield* SqlClient.SqlClient;
2464
+ const task = yield* withService(
2465
+ ({ service, launches }) =>
2466
+ Effect.gen(function* () {
2467
+ const { task } = yield* service.upsert(
2468
+ yield* webhookTaskInput({
2469
+ schedule: {
2470
+ type: "webhook",
2471
+ dedupe: { coalesceKey: "{{body}}", coalesceWindowSeconds: 10 },
2472
+ },
2473
+ }),
2474
+ );
2475
+ yield* service.triggerWebhook(requestFor(task));
2476
+ assert.equal(yield* Queue.size(launches), 0);
2477
+ return task;
2478
+ }),
2479
+ { sql, workers: new Map() },
2480
+ );
2481
+ // The first service scope is closed before the deadline advances.
2482
+ yield* TestClock.adjust("30 seconds");
2483
+ const restartedWorkers = new Map<string, Effect.Effect<void>>();
2484
+ yield* withService(
2485
+ ({ service, launches }) =>
2486
+ Effect.gen(function* () {
2487
+ yield* drainPendingDispatch(restartedWorkers);
2488
+ yield* Queue.take(launches);
2489
+ for (let i = 0; i < 3; i++) yield* restartedWorkers.get("webhook-coalescing")!;
2490
+ assert.equal(yield* Queue.size(launches), 0);
2491
+ assert.equal((yield* service.list()).tasks[0]!.runCount, 1);
2492
+ assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0);
2493
+ }),
2494
+ { sql, workers: restartedWorkers },
2495
+ );
2496
+ const secondRestartWorkers = new Map<string, Effect.Effect<void>>();
2497
+ yield* withService(
2498
+ ({ service, launches }) =>
2499
+ Effect.gen(function* () {
2500
+ yield* secondRestartWorkers.get("webhook-coalescing")!;
2501
+ assert.equal(yield* Queue.size(launches), 0);
2502
+ assert.equal(
2503
+ (yield* service.list()).tasks.find((entry) => entry.id === task.id)!.runCount,
2504
+ 1,
2505
+ );
2506
+ }),
2507
+ { sql, workers: secondRestartWorkers },
2508
+ );
2509
+ }).pipe(Effect.provide(SqlitePersistence.layerMemory)),
2510
+);
2511
+
2512
+it.effect("removes a window with a render defect and dispatches the task's next window", () => {
2513
+ const workers = new Map<string, Effect.Effect<void>>();
2514
+ return withService(
2515
+ ({ service, sql, launches }) =>
2516
+ Effect.gen(function* () {
2517
+ const { task } = yield* service.upsert(
2518
+ yield* webhookTaskInput({
2519
+ prompt: "{{body.deep}}",
2520
+ schedule: {
2521
+ type: "webhook",
2522
+ dedupe: {
2523
+ replayKey: "{{headers.delivery-id}}",
2524
+ coalesceKey: "{{headers.group}}",
2525
+ coalesceWindowSeconds: 10,
2526
+ },
2527
+ },
2528
+ }),
2529
+ );
2530
+ const request = (id: string, bodyText: string) =>
2531
+ requestFor(task, {
2532
+ bodyText,
2533
+ body: new TextEncoder().encode(bodyText),
2534
+ headers: { "delivery-id": id, group: id },
2535
+ relayDeliveryId: id,
2536
+ });
2537
+ const bad = request(
2538
+ "bad-render",
2539
+ '{"deep":' + "[".repeat(20000) + "0" + "]".repeat(20000) + "}",
2540
+ );
2541
+ assert.throws(() => renderWebhookPrompt(task.prompt, bad, "frame"), RangeError);
2542
+ const accepted = yield* service.triggerWebhook(bad);
2543
+ assert.equal(accepted._tag, "accepted");
2544
+ yield* service.triggerWebhook(request("good-render", '{"deep":"next window"}'));
2545
+ yield* TestClock.adjust("10 seconds");
2546
+ yield* drainPendingDispatch(workers);
2547
+ assert.include((yield* Queue.take(launches)).initialMessage!.text, "next window");
2548
+ assert.equal(yield* Queue.size(launches), 0);
2549
+ assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0);
2550
+ assert.equal(
2551
+ (yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys WHERE delivery_id = 'delivery:relay:bad-render'`)
2552
+ .length,
2553
+ 0,
2554
+ );
2555
+ assert.equal(
2556
+ (yield* sql`SELECT 1 FROM scheduled_task_webhook_relay_deliveries WHERE relay_delivery_id = 'bad-render'`)
2557
+ .length,
2558
+ 0,
2559
+ );
2560
+ const failed = (yield* service.getWebhookDelivery({
2561
+ id: task.id,
2562
+ deliveryId: ScheduledTaskWebhookDeliveryId.make("delivery:relay:bad-render"),
2563
+ acceptAnyOutcome: true,
2564
+ })).delivery;
2565
+ assert.equal(failed.outcome, "dispatch_failed");
2566
+ assert.equal(failed.error, "The pending webhook payload could not be prepared.");
2567
+ yield* workers.get("webhook-coalescing")!;
2568
+ assert.equal((yield* service.list()).tasks[0]!.runCount, 1);
2569
+ }),
2570
+ { workers },
2571
+ );
2572
+});
2573
+
2574
+it.effect.each([false, true])(
2575
+ "counts an interrupted immediate run beside an unrelated running window (corrupt task: %s)",
2576
+ (corrupt) => {
2577
+ const workers = new Map<string, Effect.Effect<void>>();
2578
+ const completedLaunches = new Set<string>();
2579
+ return Effect.gen(function* () {
2580
+ const gate = yield* Deferred.make<void>();
2581
+ yield* withService(
2582
+ ({ service, sql, launches }) =>
2583
+ Effect.gen(function* () {
2584
+ const { task } = yield* service.upsert(
2585
+ yield* webhookTaskInput({
2586
+ prompt: "{{body.value}}",
2587
+ schedule: {
2588
+ type: "webhook",
2589
+ dedupe: { coalesceKey: "{{headers.group}}", coalesceWindowSeconds: 10 },
2590
+ },
2591
+ }),
2592
+ );
2593
+ const window = yield* service.triggerWebhook(
2594
+ requestFor(task, { headers: { group: "old" }, bodyText: '{"value":"old"}' }),
2595
+ );
2596
+ yield* sql`CREATE TRIGGER fail_old_completion BEFORE UPDATE ON scheduled_tasks
2597
+ WHEN NEW.last_run_status = 'succeeded' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`;
2598
+ yield* TestClock.adjust("10 seconds");
2599
+ yield* drainPendingDispatch(workers);
2600
+ yield* Queue.take(launches);
2601
+ yield* sql`DROP TRIGGER fail_old_completion`;
2602
+ assert.equal(
2603
+ (yield* sql`SELECT 1 FROM scheduled_task_webhook_pending WHERE state = 'running'`)
2604
+ .length,
2605
+ 1,
2606
+ );
2607
+ const immediate = yield* service.triggerWebhook(
2608
+ requestFor(task, { bodyText: '{"value":"immediate"}' }),
2609
+ );
2610
+ yield* Queue.take(launches);
2611
+ const [before] = yield* sql<{
2612
+ last_run_webhook_delivery_id: string;
2613
+ model_selection_json: string;
2614
+ run_count: number;
2615
+ }>`SELECT last_run_webhook_delivery_id, model_selection_json, run_count FROM scheduled_tasks WHERE task_id = ${task.id}`;
2616
+ assert.equal(
2617
+ before!.last_run_webhook_delivery_id,
2618
+ immediate._tag === "accepted" ? immediate.deliveryId : "",
2619
+ );
2620
+ assert.notEqual(
2621
+ before!.last_run_webhook_delivery_id,
2622
+ window._tag === "accepted" ? window.deliveryId : "",
2623
+ );
2624
+ assert.equal(before!.run_count, 0);
2625
+ if (corrupt)
2626
+ yield* sql`UPDATE scheduled_tasks SET model_selection_json = 'broken' WHERE task_id = ${task.id}`;
2627
+ const restartedWorkers = new Map<string, Effect.Effect<void>>();
2628
+ yield* withService(
2629
+ ({ service: restarted, launches: recoveredLaunches }) =>
2630
+ Effect.gen(function* () {
2631
+ const [recovered] = yield* sql<{
2632
+ run_count: number;
2633
+ last_run_status: string;
2634
+ }>`SELECT run_count, last_run_status FROM scheduled_tasks WHERE task_id = ${task.id}`;
2635
+ assert.equal(recovered!.run_count, 1);
2636
+ assert.equal(recovered!.last_run_status, "failed");
2637
+ if (corrupt)
2638
+ yield* sql`UPDATE scheduled_tasks SET model_selection_json = ${before!.model_selection_json} WHERE task_id = ${task.id}`;
2639
+ yield* drainPendingDispatch(restartedWorkers);
2640
+ assert.equal(yield* Queue.size(recoveredLaunches), 0);
2641
+ assert.equal((yield* restarted.list()).tasks[0]!.runCount, 2);
2642
+ assert.equal(
2643
+ (yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length,
2644
+ 0,
2645
+ );
2646
+ yield* restartedWorkers.get("webhook-coalescing")!;
2647
+ assert.equal((yield* restarted.list()).tasks[0]!.runCount, 2);
2648
+ }),
2649
+ { sql, workers: restartedWorkers, completedLaunches },
2650
+ );
2651
+ }),
2652
+ {
2653
+ workers,
2654
+ completedLaunches,
2655
+ gateForTask: (input) =>
2656
+ input.initialMessage?.text.includes(">immediate<") ? gate : undefined,
2657
+ },
2658
+ );
2659
+ });
2660
+ },
2661
+);
2662
+
2663
+it.effect("checks filters after the signature and max age, and runs only matching deliveries", () =>
2664
+ withService(({ service, launches }) =>
2665
+ Effect.gen(function* () {
2666
+ const { task } = yield* service.upsert(
2667
+ yield* webhookTaskInput({
2668
+ schedule: {
2669
+ type: "webhook",
2670
+ signature: {
2671
+ header: "x-hub-signature-256",
2672
+ encoding: "hex",
2673
+ prefix: "sha256=",
2674
+ secret: "s3cret",
2675
+ },
2676
+ maxDeliveryAgeMinutes: 30,
2677
+ filters: [
2678
+ { path: "headers.x-github-event", op: "equals", value: "pull_request" },
2679
+ { path: "body.action", op: "in", value: ["opened", "reopened"] },
2680
+ ],
2681
+ },
2682
+ }),
2683
+ );
2684
+ const delivery = (
2685
+ event: string,
2686
+ action: string,
2687
+ options: { readonly signed?: boolean; readonly receivedAt?: string } = {},
2688
+ ) => {
2689
+ const body = new TextEncoder().encode(
2690
+ JSON.stringify({ action, pull_request: { url: "https://github.com/org/repo/pull/45" } }),
2691
+ );
2692
+ const signature = `sha256=${NodeCrypto.createHmac("sha256", "s3cret").update(body).digest("hex")}`;
2693
+ return requestFor(task, {
2694
+ headers: {
2695
+ "content-type": "application/json",
2696
+ "x-github-event": event,
2697
+ ...(options.signed === false ? {} : { "x-hub-signature-256": signature }),
2698
+ },
2699
+ body,
2700
+ bodyText: new TextDecoder().decode(body),
2701
+ ...(options.receivedAt === undefined
2702
+ ? {}
2703
+ : { relayDeliveryId: "old", receivedAt: options.receivedAt }),
2704
+ });
2705
+ };
2706
+ const now = yield* DateTime.now;
2707
+ const old = DateTime.formatIso(DateTime.subtract(now, { minutes: 31 }));
2708
+
2709
+ // A request failing a filter is still refused first for its signature or age.
2710
+ const unsigned = yield* service.triggerWebhook(delivery("push", "opened", { signed: false }));
2711
+ assert.equal(unsigned._tag, "rejected_signature");
2712
+ const tooOld = yield* service.triggerWebhook(delivery("push", "opened", { receivedAt: old }));
2713
+ assert.equal(tooOld._tag, "expired");
2714
+
2715
+ const wrongEvent = yield* service.triggerWebhook(delivery("push", "opened"));
2716
+ const wrongAction = yield* service.triggerWebhook(delivery("pull_request", "closed"));
2717
+ assert.deepInclude(wrongEvent, { _tag: "accepted", outcome: "filtered" });
2718
+ assert.deepInclude(wrongAction, { _tag: "accepted", outcome: "filtered" });
2719
+ assert.equal(yield* Queue.size(launches), 0);
2720
+ const [listed] = (yield* service.list()).tasks;
2721
+ assert.equal(listed?.runCount, 0);
2722
+ assert.equal(listed?.lastRunStatus, "never");
2723
+
2724
+ const matched = yield* service.triggerWebhook(delivery("pull_request", "opened"));
2725
+ assert.deepInclude(matched, { _tag: "accepted", outcome: "accepted" });
2726
+ assert.include((yield* Queue.take(launches)).initialMessage?.text ?? "", "pull/45");
2727
+
2728
+ const deliveries = (yield* service.listWebhookDeliveries({
2729
+ id: task.id,
2730
+ acceptAnyOutcome: true,
2731
+ })).deliveries;
2732
+ const filtered = deliveries
2733
+ .filter((entry) => entry.outcome === "filtered")
2734
+ .map((entry) => [entry.signatureVerified, entry.error]);
2735
+ assert.sameDeepMembers(filtered, [
2736
+ [true, "Filter 1 (headers.x-github-event) did not match."],
2737
+ [true, "Filter 2 (body.action) did not match."],
2738
+ ]);
2739
+ }),
2740
+ ),
2741
+);
2742
+
2743
+const pullRequestFilterInput = () =>
2744
+ webhookTaskInput({
2745
+ schedule: {
2746
+ type: "webhook",
2747
+ filters: [{ path: "headers.x-github-event", op: "equals", value: "pull_request" }],
2748
+ },
2749
+ });
2750
+const githubEvent = (event: string) => ({
2751
+ headers: { "content-type": "application/json", "x-github-event": event },
2752
+});
2753
+
2754
+it.effect("filtered deliveries take no run slot and log at most 10 rows a minute", () =>
2755
+ withService(({ service, launches }) =>
2756
+ Effect.gen(function* () {
2757
+ const { task } = yield* service.upsert(yield* pullRequestFilterInput());
2758
+ const ignored = yield* Effect.forEach(Array.from({ length: 70 }), () =>
2759
+ service.triggerWebhook(requestFor(task, githubEvent("push"))),
2760
+ );
2761
+ assert.isTrue(
2762
ignored.every((result) => result._tag === "accepted" && result.outcome === "filtered"),
2763
);
2764
const matched = yield* service.triggerWebhook(requestFor(task, githubEvent("pull_request")));
@@ -1157,3 +2863,75 @@ it.effect("leaves filtered rows out of the list for clients that only decode old
2863
}),
2864
),
2865
);
2866
+
2867
+it.effect("filters run before replay and coalescing without claiming ignored deliveries", () => {
2868
+ const workers = new Map<string, Effect.Effect<void>>();
2869
+ return withService(
2870
+ ({ service, launches, sql }) =>
2871
+ Effect.gen(function* () {
2872
+ const { task } = yield* service.upsert(
2873
+ yield* webhookTaskInput({
2874
+ prompt: "{{body.value}}",
2875
+ schedule: {
2876
+ type: "webhook",
2877
+ filters: [{ path: "body.run", op: "equals", value: "true" }],
2878
+ dedupe: {
2879
+ replayKey: "{{headers.delivery-id}}",
2880
+ coalesceKey: "{{body.group}}",
2881
+ coalesceWindowSeconds: 10,
2882
+ },
2883
+ },
2884
+ }),
2885
+ );
2886
+ const request = (id: string, run: boolean, value: string) => {
2887
+ const bodyText = JSON.stringify({ run, group: "B", value });
2888
+ return requestFor(task, {
2889
+ headers: { "delivery-id": id },
2890
+ bodyText,
2891
+ body: new TextEncoder().encode(bodyText),
2892
+ relayDeliveryId: id,
2893
+ });
2894
+ };
2895
+ const ignored = yield* service.triggerWebhook(request("A", false, "ignored"));
2896
+ assert.deepInclude(ignored, { _tag: "accepted", outcome: "filtered" });
2897
+ assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`).length, 0);
2898
+ assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0);
2899
+ assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_relay_deliveries`).length, 0);
2900
+ const filteredId = ignored._tag === "accepted" ? ignored.deliveryId : assert.fail();
2901
+ const oldClient = yield* service
2902
+ .getWebhookDelivery({ id: task.id, deliveryId: filteredId })
2903
+ .pipe(Effect.result);
2904
+ assert.equal(oldClient._tag, "Failure");
2905
+ const currentClient = yield* service.getWebhookDelivery({
2906
+ id: task.id,
2907
+ deliveryId: filteredId,
2908
+ acceptAnyOutcome: true,
2909
+ });
2910
+ assert.equal(currentClient.delivery.outcome, "filtered");
2911
+ assert.deepInclude(yield* service.triggerWebhook(request("A", true, "first")), {
2912
+ _tag: "accepted",
2913
+ outcome: "accepted",
2914
+ });
2915
+ assert.deepInclude(
2916
+ yield* service.triggerWebhook({
2917
+ ...request("A", false, "ignored replay"),
2918
+ relayDeliveryId: "ignored-replay",
2919
+ }),
2920
+ { _tag: "accepted", outcome: "filtered" },
2921
+ );
2922
+ assert.deepInclude(yield* service.triggerWebhook(request("B", false, "ignored update")), {
2923
+ _tag: "accepted",
2924
+ outcome: "filtered",
2925
+ });
2926
+ assert.deepInclude(yield* service.triggerWebhook(request("B", true, "latest")), {
2927
+ _tag: "accepted",
2928
+ outcome: "coalesced",
2929
+ });
2930
+ yield* TestClock.adjust("10 seconds");
2931
+ yield* drainPendingDispatch(workers);
2932
+ assert.include((yield* Queue.take(launches)).initialMessage!.text, "latest");
2933
+ assert.equal(yield* Queue.size(launches), 0);
2934
+ }),
2935
+ { workers },
2936
+ );
2937
+});
apps/server/src/scheduledTasks/webhookDedupe.ts
new
+128
@@ -0,0 +1,128 @@
1
+// @effect-diagnostics-next-line nodeBuiltinImport:off -- Keys need a deterministic digest, not random bytes.
2
+import * as NodeCrypto from "node:crypto";
3
+import { ScheduledTask, ScheduledTaskError } from "@t3tools/contracts";
4
+import * as Effect from "effect/Effect";
5
+import * as Schema from "effect/Schema";
6
+import * as SqlClient from "effect/sql/SqlClient";
7
+import { renderWebhookPrompt, type WebhookRequest } from "./webhookTemplate.ts";
8
+
9
+function key(template: string | undefined, request: WebhookRequest): string {
10
+ if (template === undefined) return "";
11
+ const rendered = renderWebhookPrompt(template, request, "", true);
12
+ return rendered.missing.length > 0 || rendered.prompt === ""
13
+ ? ""
14
+ : NodeCrypto.createHash("sha256").update(rendered.prompt).digest("hex");
15
+}
16
+
17
+export const releaseWebhookReplay = Effect.fn("ScheduledTaskService.releaseWebhookReplay")(
18
+ function* (deliveryId: string) {
19
+ const sql = yield* SqlClient.SqlClient;
20
+ yield* sql`DELETE FROM scheduled_task_webhook_replay_keys WHERE delivery_id = ${deliveryId}`;
21
+ },
22
+);
23
+
24
+const hasReplayKey = Effect.fn(function* (taskId: string, replayKey: string, now: number) {
25
+ const sql = yield* SqlClient.SqlClient;
26
+ return (
27
+ replayKey !== "" &&
28
+ (yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys
29
+ WHERE task_id = ${taskId} AND replay_key = ${replayKey} AND expires_at > ${now}`).length > 0
30
+ );
31
+});
32
+const claimReplayKey = Effect.fn(function* (
33
+ taskId: string,
34
+ replayKey: string,
35
+ deliveryId: string,
36
+ now: number,
37
+) {
38
+ if (replayKey === "") return;
39
+ const sql = yield* SqlClient.SqlClient;
40
+ yield* sql`DELETE FROM scheduled_task_webhook_replay_keys WHERE expires_at <= ${now}`;
41
+ yield* sql`INSERT INTO scheduled_task_webhook_replay_keys (task_id, replay_key, delivery_id, expires_at)
42
+ VALUES (${taskId}, ${replayKey}, ${deliveryId}, ${now + 7 * 24 * 60 * 60 * 1000})`;
43
+});
44
+
45
+export const acceptWebhookDedupe = Effect.fn("ScheduledTaskService.acceptWebhookDedupe")(function* (
46
+ task: ScheduledTask,
47
+ request: WebhookRequest,
48
+ deliveryId: string,
49
+ now: number,
50
+ reserve: Effect.Effect<void, "first_rejected" | "rejected">,
51
+ reserveCoalescedLog: Effect.Effect<boolean>,
52
+ checkCapacity: Effect.Effect<void, "queue_full" | ScheduledTaskError>,
53
+ replaceDeliveryRequest: (ownerId: string) => Effect.Effect<void, ScheduledTaskError>,
54
+) {
55
+ const sql = yield* SqlClient.SqlClient;
56
+ const dedupe = task.schedule.type === "webhook" ? task.schedule.dedupe : undefined;
57
+ if (dedupe === undefined) return "run" as const;
58
+ const replayKey = key(dedupe.replayKey, request);
59
+ if (yield* hasReplayKey(task.id, replayKey, now)) return "duplicate" as const;
60
+ const coalesceKey = key(dedupe.coalesceKey, request);
61
+ if (coalesceKey === "") return "run" as const;
62
+ const pending = yield* sql<{
63
+ delivery_id: string;
64
+ }>`SELECT delivery_id FROM scheduled_task_webhook_pending
65
+ WHERE task_id = ${task.id} AND coalesce_key = ${coalesceKey} AND state = 'pending' AND due_at > ${now}`;
66
+ const logCoalesced = pending.length > 0 ? yield* reserveCoalescedLog : true;
67
+ if (pending.length === 0) {
68
+ yield* checkCapacity;
69
+ yield* reserve;
70
+ }
71
+ // Coalesced requests retain their validation slot, bounding writes without dropping accepted payloads.
72
+ yield* claimReplayKey(task.id, replayKey, pending[0]?.delivery_id ?? deliveryId, now);
73
+ const requestJson = yield* encodePendingRequest({
74
+ method: request.method,
75
+ path: request.path,
76
+ query: request.query,
77
+ headers: request.headers,
78
+ bodyText: request.bodyText,
79
+ });
80
+ if (pending.length > 0) {
81
+ yield* sql`UPDATE scheduled_task_webhook_pending SET request_json = ${requestJson},
82
+ delivery_ids_json = json_insert(delivery_ids_json, '$[#]', ${deliveryId}) WHERE delivery_id = ${pending[0]!.delivery_id}`;
83
+ yield* replaceDeliveryRequest(pending[0]!.delivery_id);
84
+ return logCoalesced ? ("coalesced" as const) : ("coalesced_unlogged" as const);
85
+ }
86
+ // Expired windows are immutable, even if the scheduler has not dispatched them yet.
87
+ yield* sql`UPDATE scheduled_task_webhook_pending SET state = 'dispatching'
88
+ WHERE task_id = ${task.id} AND coalesce_key = ${coalesceKey} AND state = 'pending' AND due_at <= ${now}`;
89
+ yield* sql`INSERT INTO scheduled_task_webhook_pending (delivery_id, task_id, coalesce_key, due_at, task_json, request_json, delivery_ids_json)
90
+ VALUES (${deliveryId}, ${task.id}, ${coalesceKey}, ${now + (dedupe.coalesceWindowSeconds ?? 120) * 1000}, ${yield* encodePendingTask(task)}, ${requestJson}, json_array(${deliveryId}))`;
91
+ return "pending" as const;
92
+});
93
+
94
+export const acceptImmediateWebhookDedupe = Effect.fn(
95
+ "ScheduledTaskService.acceptImmediateWebhookDedupe",
96
+)(function* (
97
+ task: ScheduledTask,
98
+ request: WebhookRequest,
99
+ deliveryId: string,
100
+ now: number,
101
+ reserve: Effect.Effect<void, "first_rejected" | "rejected" | "queue_full" | ScheduledTaskError>,
102
+) {
103
+ const replayKey = key(
104
+ task.schedule.type === "webhook" ? task.schedule.dedupe?.replayKey : undefined,
105
+ request,
106
+ );
107
+ if (yield* hasReplayKey(task.id, replayKey, now)) return "duplicate" as const;
108
+ yield* reserve;
109
+ yield* claimReplayKey(task.id, replayKey, deliveryId, now);
110
+ return "run" as const;
111
+});
112
+
113
+const PendingTaskJson = Schema.fromJsonString(ScheduledTask);
114
+const encodePendingTask = Schema.encodeEffect(PendingTaskJson);
115
+export const decodePendingTask = Schema.decodeUnknownEffect(PendingTaskJson);
116
+const PendingRequestJson = Schema.fromJsonString(
117
+ Schema.Struct({
118
+ method: Schema.String,
119
+ path: Schema.String,
120
+ query: Schema.String,
121
+ headers: Schema.Record(Schema.String, Schema.String),
122
+ bodyText: Schema.String,
123
+ }),
124
+);
125
+const encodePendingRequest = Schema.encodeEffect(PendingRequestJson);
126
+export const decodePendingRequest = Schema.decodeUnknownEffect(PendingRequestJson);
127
+export const dedupeError = (cause: unknown) =>
128
+ new ScheduledTaskError({ message: "Could not persist webhook dedupe state.", cause });
apps/server/src/scheduledTasks/webhookRoute.test.ts
+1
@@ -236,6 +236,7 @@ describe("webhook route", () => {
236
const response = await handler(post("/api/hooks/id/tok", "{}"));
237
expect(response.status).toBe(status);
238
expect(response.headers.get("x-t3-hook-outcome")).toBe(outcome);
239
+ if (status === 429) expect(await response.json()).toEqual({ error: outcome });
240
} finally {
241
await dispose();
242
}
apps/server/src/scheduledTasks/webhookRoute.ts
+1
-1
@@ -118,7 +118,7 @@ export const layer = HttpApiBuilder.group(
118
case "disabled":
119
return json(409, { error: "hook_disabled" }, "disabled");
120
case "rate_limited":
121
- return json(429, { error: "rate_limited" }, result.outcome);
121
+ return json(429, { error: result.outcome }, result.outcome);
122
case "expired":
123
return json(410, { error: "delivery_too_old" }, "expired");
124
case "error":
apps/server/src/scheduledTasks/webhookTemplate.ts
+3
-1
@@ -150,6 +150,7 @@ export function renderWebhookPrompt(
150
template: string,
151
request: WebhookRequest,
152
frameId: string,
153
+ raw = false,
154
): RenderedWebhookPrompt {
155
const missing: string[] = [];
156
const resolve = webhookExpressionResolver(request);
@@ -160,12 +161,13 @@ export function renderWebhookPrompt(
161
const expression = rawExpression.trim();
162
const value = resolve(expression);
163
const text = value === undefined || value === null ? undefined : stringify(value);
163
- if (text === undefined) {
164
+ if (text === undefined || (raw && text === "")) {
165
if (!missing.includes(expression)) missing.push(expression);
166
return "";
167
}
168
// An empty value carries no request data, so it needs no frame or note.
169
if (text === "") return "";
170
+ if (raw) return text;
171
framed = true;
172
return `<${tag}>${text.replace(MENTION, "$&\u2060")}</${tag}>`;
173
});
apps/web/src/components/settings/ScheduledTasksSettings.tsx
+87
-30
@@ -22,7 +22,6 @@ import type {
22
ScheduledTaskId,
23
ScheduledTaskSchedule,
24
ScheduledTaskUpsertInput,
25
- ScheduledTaskWebhookDeliveryOutcome,
25
ScheduledTaskWebhookDeliverySummary,
26
ThreadId,
27
} from "@t3tools/contracts";
@@ -81,6 +80,8 @@ import { SettingsGroup } from "./SettingsGroup";
80
import { settingsScopeEnvironmentLabel, settingsScopeEnvironmentLabels } from "./settingsScopeAxis";
81
import {
82
WEBHOOK_SIGNATURE_DEFAULTS,
83
+ webhookDraftErrors,
84
+ webhookDeliveryLabel,
85
matchesScheduledTaskScope,
86
scheduleFromDraft,
87
scheduledTaskDefaultModel,
@@ -152,6 +153,9 @@ const EMPTY_DRAFT: DraftState = {
153
...WEBHOOK_SIGNATURE_DEFAULTS,
154
signatureSecret: "",
155
maxDeliveryAgeMinutes: "",
156
+ replayKey: "",
157
+ coalesceKey: "",
158
+ coalesceWindowSeconds: "",
159
filters: [],
160
savedFilters: [],
161
};
@@ -328,26 +332,15 @@ export function ScheduledRelativeLabel(props: { value: string | null }) {
332
return relativeLabel(props.value, Date.parse(`${nowMinute}:00.000Z`));
333
}
334
331
-const DELIVERY_OUTCOME_LABELS: Record<ScheduledTaskWebhookDeliveryOutcome, string> = {
332
- accepted: "Ran",
333
- dispatch_failed: "Run failed",
334
- rejected_signature: "Bad signature",
335
- disabled: "Task paused",
336
- rate_limited: "Rate limited",
337
- expired: "Too old",
338
- filtered: "Filtered",
339
-};
340
-
341
-// A newer server can report an outcome this client has no label for.
342
-function deliveryOutcomeLabel(outcome: string): string {
343
- return Object.hasOwn(DELIVERY_OUTCOME_LABELS, outcome)
344
- ? DELIVERY_OUTCOME_LABELS[outcome as ScheduledTaskWebhookDeliveryOutcome]
345
- : outcome.replaceAll("_", " ");
346
-}
347
-
335
function deliveryOutcomeVariant(outcome: string) {
336
if (outcome === "accepted") return "success";
350
- if (outcome === "disabled" || outcome === "rate_limited" || outcome === "expired") {
337
+ if (
338
+ outcome === "duplicate" ||
339
+ outcome === "coalesced" ||
340
+ outcome === "disabled" ||
341
+ outcome === "rate_limited" ||
342
+ outcome === "expired"
343
+ ) {
344
return "warning";
345
}
346
if (outcome === "dispatch_failed" || outcome === "rejected_signature") return "error";
@@ -822,7 +815,7 @@ function WebhookDeliveriesDialog({
815
? null
816
: serverEnvironment.scheduledTaskWebhookDelivery({
817
environmentId,
825
- input: { id: task.id, deliveryId: selectedId },
818
+ input: { id: task.id, deliveryId: selectedId, acceptAnyOutcome: true },
819
}),
820
);
821
const deliveries = deliveriesQuery.data?.deliveries ?? null;
@@ -850,8 +843,12 @@ function WebhookDeliveriesDialog({
843
) : selected ? (
844
<div className="space-y-4">
845
<div className="flex flex-wrap items-center gap-2 text-sm">
853
- <Badge variant={deliveryOutcomeVariant(selected.outcome)}>
854
- {deliveryOutcomeLabel(selected.outcome)}
846
+ <Badge
847
+ variant={deliveryOutcomeVariant(
848
+ selected.pendingWindow ? "coalesced" : selected.outcome,
849
+ )}
850
+ >
851
+ {webhookDeliveryLabel(selected)}
852
</Badge>
853
<span>
854
{selected.method} · <ScheduledRelativeLabel value={selected.receivedAt} />
@@ -871,8 +868,13 @@ function WebhookDeliveriesDialog({
868
{selected.error}
869
</p>
870
) : null}
874
- <DeliveryBlock title="Prompt sent to the agent">
875
- {selected.renderedPrompt ?? "No run was started for this request."}
871
+ <DeliveryBlock
872
+ title={selected.pendingWindow ? "Scheduled run" : "Prompt sent to the agent"}
873
+ >
874
+ {selected.renderedPrompt ??
875
+ (selected.pendingWindow
876
+ ? "Run scheduled when the window closes; no run started yet."
877
+ : "No run was started for this request.")}
878
</DeliveryBlock>
879
{selected.missingFields.length > 0 ? (
880
<p className="text-sm text-muted-foreground">
@@ -906,8 +908,12 @@ function WebhookDeliveriesDialog({
908
className="flex w-full flex-wrap items-center gap-2 py-2 text-left text-sm hover:bg-accent/40"
909
onClick={() => setSelectedId(delivery.id)}
910
>
909
- <Badge variant={deliveryOutcomeVariant(delivery.outcome)}>
910
- {deliveryOutcomeLabel(delivery.outcome)}
911
+ <Badge
912
+ variant={deliveryOutcomeVariant(
913
+ delivery.pendingWindow ? "coalesced" : delivery.outcome,
914
+ )}
915
+ >
916
+ {webhookDeliveryLabel(delivery)}
917
</Badge>
918
<span>{delivery.method}</span>
919
<span className="text-muted-foreground">
@@ -1193,10 +1199,7 @@ function ScheduledTaskEditorForm({
1199
}
1200
const schedule = scheduleFromDraft(draft);
1201
if (schedule === null) {
1196
- reportFailure(
1197
- "Invalid age limit",
1198
- `Enter whole minutes from 1 to ${MAX_WEBHOOK_DELIVERY_AGE_MINUTES}, or leave it blank.`,
1199
- );
1202
+ for (const error of webhookDraftErrors(draft)) reportFailure(error.title, error.description);
1203
return;
1204
}
1205
if (
@@ -1543,6 +1546,60 @@ function ScheduledTaskEditorForm({
1546
filters={draft.filters}
1547
onChange={(filters) => setDraft((current) => ({ ...current, filters }))}
1548
/>
1549
+ <Field
1550
+ label="Replay key"
1551
+ hint="template, optional"
1552
+ htmlFor="scheduled-task-replay-key"
1553
+ >
1554
+ <Input
1555
+ id="scheduled-task-replay-key"
1556
+ placeholder="{{headers.delivery-id}}"
1557
+ value={draft.replayKey}
1558
+ onChange={(event) =>
1559
+ setDraft((current) => ({ ...current, replayKey: event.target.value }))
1560
+ }
1561
+ />
1562
+ </Field>
1563
+ <Field
1564
+ label="Coalesce key"
1565
+ hint="template, optional"
1566
+ htmlFor="scheduled-task-coalesce-key"
1567
+ >
1568
+ <Input
1569
+ id="scheduled-task-coalesce-key"
1570
+ placeholder="{{body.id}}"
1571
+ value={draft.coalesceKey}
1572
+ onChange={(event) =>
1573
+ setDraft((current) => ({ ...current, coalesceKey: event.target.value }))
1574
+ }
1575
+ />
1576
+ </Field>
1577
+ <Field
1578
+ label="Coalesce window"
1579
+ hint="seconds, 10–3600"
1580
+ htmlFor="scheduled-task-coalesce-window"
1581
+ >
1582
+ <Input
1583
+ id="scheduled-task-coalesce-window"
1584
+ type="number"
1585
+ nativeInput
1586
+ min={10}
1587
+ max={3600}
1588
+ placeholder="120"
1589
+ value={draft.coalesceWindowSeconds}
1590
+ onChange={(event) =>
1591
+ setDraft((current) => ({
1592
+ ...current,
1593
+ coalesceWindowSeconds: event.target.value,
1594
+ }))
1595
+ }
1596
+ />
1597
+ </Field>
1598
+ <p className="text-xs text-muted-foreground">
1599
+ Replay keys suppress repeat deliveries for 7 days. Coalescing runs the latest
1600
+ payload once when the window from the first delivery closes. Empty keys run
1601
+ normally.
1602
+ </p>
1603
<div className="flex items-center justify-between gap-4">
1604
<div className="min-w-0 space-y-1">
1605
<Label htmlFor="scheduled-task-signature">Require signature</Label>
apps/web/src/components/settings/scheduledTasksSettings.logic.test.ts
+98
@@ -20,6 +20,8 @@ import {
20
scheduledTaskDefaultModel,
21
matchesScheduledTaskScope,
22
scheduleFromDraft,
23
+ webhookDraftErrors,
24
+ webhookDeliveryLabel,
25
taskToDraft,
26
} from "./scheduledTasksSettings.logic";
27
@@ -175,6 +177,43 @@ describe("webhook scheduled tasks", () => {
177
webhook: { path: "/api/hooks/legacy-task/token", url: null, hasSecret: true },
178
};
179
180
+ it("round-trips dedupe settings, clears them, and rejects invalid windows", () => {
181
+ const dedupe = {
182
+ replayKey: "{{headers.delivery-id}}",
183
+ coalesceKey: "{{body.id}}",
184
+ coalesceWindowSeconds: 10,
185
+ };
186
+ const draft = taskToDraft({
187
+ ...webhookTask,
188
+ schedule: { type: "webhook", signature: null, dedupe },
189
+ });
190
+ expect(scheduleFromDraft(draft)).toMatchObject({ dedupe });
191
+ expect(
192
+ scheduleFromDraft({
193
+ ...draft,
194
+ replayKey: " \n ",
195
+ coalesceKey: " \t ",
196
+ coalesceWindowSeconds: "",
197
+ }),
198
+ ).toMatchObject({ dedupe: {} });
199
+ expect(
200
+ scheduleFromDraft({
201
+ ...draft,
202
+ replayKey: ` ${dedupe.replayKey} `,
203
+ coalesceKey: `\t${dedupe.coalesceKey}\n`,
204
+ }),
205
+ ).toMatchObject({ dedupe });
206
+ expect(
207
+ scheduleFromDraft({ ...draft, replayKey: "", coalesceKey: "", coalesceWindowSeconds: "" }),
208
+ ).toMatchObject({ dedupe: {} });
209
+ expect(scheduleFromDraft({ ...draft, coalesceWindowSeconds: "" })).toMatchObject({
210
+ dedupe: { replayKey: dedupe.replayKey, coalesceKey: dedupe.coalesceKey },
211
+ });
212
+ for (const coalesceWindowSeconds of ["9", "3601", "1.5", "abc"]) {
213
+ expect(scheduleFromDraft({ ...draft, coalesceWindowSeconds })).toBeNull();
214
+ }
215
+ });
216
+
217
it("keeps a stored secret when the secret field is left blank", () => {
218
const draft = taskToDraft(webhookTask);
219
expect(draft.scheduleMode).toBe("webhook");
@@ -183,11 +222,13 @@ describe("webhook scheduled tasks", () => {
222
type: "webhook",
223
signature,
224
maxDeliveryAgeMinutes: null,
225
+ dedupe: {},
226
});
227
expect(scheduleFromDraft({ ...draft, signatureSecret: " new " })).toEqual({
228
type: "webhook",
229
signature: { ...signature, secret: "new" },
230
maxDeliveryAgeMinutes: null,
231
+ dedupe: {},
232
});
233
});
234
@@ -199,6 +240,7 @@ describe("webhook scheduled tasks", () => {
240
type: "webhook",
241
signature: null,
242
maxDeliveryAgeMinutes: null,
243
+ dedupe: {},
244
});
245
});
246
@@ -212,6 +254,7 @@ describe("webhook scheduled tasks", () => {
254
for (const blank of ["", " "]) {
255
expect(scheduleFromDraft({ ...draft, maxDeliveryAgeMinutes: blank })).toMatchObject({
256
maxDeliveryAgeMinutes: null,
257
+ dedupe: {},
258
});
259
}
260
for (const invalid of ["0", "-5", "1.5", "abc", "1441"]) {
@@ -339,3 +382,58 @@ describe("scheduled task model defaults", () => {
382
).toBeNull();
383
});
384
});
385
+
386
+it("rejects dedupe templates without a request placeholder", () => {
387
+ const draft = taskToDraft({ ...legacyTask, schedule: { type: "webhook", signature: null } });
388
+ expect(scheduleFromDraft({ ...draft, replayKey: "literal" })).toBeNull();
389
+ expect(scheduleFromDraft({ ...draft, coalesceKey: "{{body.bad" })).toBeNull();
390
+});
391
+
392
+describe("webhook delivery labels and validation messages", () => {
393
+ it("preserves Ran for legacy and dispatched deliveries and distinguishes pending windows", () => {
394
+ expect(webhookDeliveryLabel({ outcome: "accepted" })).toBe("Ran");
395
+ expect(webhookDeliveryLabel({ outcome: "accepted", pendingWindow: false })).toBe("Ran");
396
+ expect(webhookDeliveryLabel({ outcome: "accepted", pendingWindow: true })).toBe(
397
+ "Opened window",
398
+ );
399
+ });
400
+
401
+ it("keeps the existing age-limit title and body", () => {
402
+ const draft = taskToDraft({ ...legacyTask, schedule: { type: "webhook", signature: null } });
403
+ expect(
404
+ webhookDraftErrors({ ...draft, maxDeliveryAgeMinutes: "0", replayKey: "literal" }),
405
+ ).toEqual([
406
+ {
407
+ title: "Invalid age limit",
408
+ description: "Enter whole minutes from 1 to 1440, or leave it blank.",
409
+ },
410
+ ]);
411
+ });
412
+
413
+ it.each([
414
+ [{ replayKey: "literal" }, "Replay key must contain a valid request placeholder."],
415
+ [{ coalesceKey: "{{body.bad" }, "Coalesce key must contain a valid request placeholder."],
416
+ [{ coalesceWindowSeconds: "9" }, "Coalesce window must be whole seconds from 10 to 3600."],
417
+ ])("names only the invalid webhook field for %o", (invalid, description) => {
418
+ const draft = taskToDraft({ ...legacyTask, schedule: { type: "webhook", signature: null } });
419
+ expect(webhookDraftErrors({ ...draft, ...invalid })).toEqual([
420
+ { title: "Invalid webhook settings", description },
421
+ ]);
422
+ });
423
+
424
+ it("reports separate short errors when several dedupe fields fail", () => {
425
+ const draft = taskToDraft({ ...legacyTask, schedule: { type: "webhook", signature: null } });
426
+ expect(
427
+ webhookDraftErrors({
428
+ ...draft,
429
+ replayKey: "bad",
430
+ coalesceKey: "bad",
431
+ coalesceWindowSeconds: "0",
432
+ }).map((error) => error.description),
433
+ ).toEqual([
434
+ "Replay key must contain a valid request placeholder.",
435
+ "Coalesce key must contain a valid request placeholder.",
436
+ "Coalesce window must be whole seconds from 10 to 3600.",
437
+ ]);
438
+ });
439
+});
apps/web/src/components/settings/scheduledTasksSettings.logic.ts
+73
@@ -1,5 +1,7 @@
1
+import { isWebhookDedupeTemplate } from "@t3tools/contracts";
2
import {
3
EnvironmentId,
4
+ MAX_WEBHOOK_DELIVERY_AGE_MINUTES,
5
type ProjectId,
6
ScheduledTaskId,
7
type ScheduledTask,
@@ -83,6 +85,9 @@ export interface DraftState {
85
/** Write-only: empty keeps the secret already stored on the server. */
86
readonly signatureSecret: string;
87
readonly maxDeliveryAgeMinutes: string;
88
+ readonly replayKey: string;
89
+ readonly coalesceKey: string;
90
+ readonly coalesceWindowSeconds: string;
91
readonly filters: ReadonlyArray<WebhookFilterDraft>;
92
/** The filter rows the form opened with. */
93
readonly savedFilters: ReadonlyArray<WebhookFilterDraft>;
@@ -94,9 +99,62 @@ export const WEBHOOK_SIGNATURE_DEFAULTS = {
99
signaturePrefix: "sha256=",
100
} as const;
101
102
+export function webhookDraftErrors(draft: DraftState) {
103
+ if (draft.scheduleMode !== "webhook") return [];
104
+ if (parseMaxDeliveryAge(draft.maxDeliveryAgeMinutes) === undefined)
105
+ return [
106
+ {
107
+ title: "Invalid age limit",
108
+ description: `Enter whole minutes from 1 to ${MAX_WEBHOOK_DELIVERY_AGE_MINUTES}, or leave it blank.`,
109
+ },
110
+ ];
111
+ const errors: Array<{ title: string; description: string }> = [];
112
+ for (const [label, value] of [
113
+ ["Replay key", draft.replayKey],
114
+ ["Coalesce key", draft.coalesceKey],
115
+ ] as const) {
116
+ if (value.trim() && !isWebhookDedupeTemplate(value.trim()))
117
+ errors.push({
118
+ title: "Invalid webhook settings",
119
+ description: `${label} must contain a valid request placeholder.`,
120
+ });
121
+ }
122
+ const window = draft.coalesceWindowSeconds.trim();
123
+ if (window && (!Number.isInteger(Number(window)) || Number(window) < 10 || Number(window) > 3600))
124
+ errors.push({
125
+ title: "Invalid webhook settings",
126
+ description: "Coalesce window must be whole seconds from 10 to 3600.",
127
+ });
128
+ return errors;
129
+}
130
+
131
+export function webhookDeliveryLabel(delivery: {
132
+ readonly outcome: string;
133
+ readonly pendingWindow?: boolean | undefined;
134
+}) {
135
+ if (delivery.outcome === "accepted") return delivery.pendingWindow ? "Opened window" : "Ran";
136
+ const labels: Record<string, string> = {
137
+ dispatch_failed: "Run failed",
138
+ duplicate: "Duplicate",
139
+ coalesced: "Coalesced",
140
+ filtered: "Filtered",
141
+ rejected_signature: "Bad signature",
142
+ disabled: "Task paused",
143
+ rate_limited: "Rate limited",
144
+ expired: "Too old",
145
+ };
146
+ return labels[delivery.outcome] ?? delivery.outcome.replaceAll("_", " ");
147
+}
148
+
149
export function scheduleFromDraft(draft: DraftState): ScheduledTaskUpsertSchedule | null {
150
if (draft.scheduleMode === "webhook") {
151
+ if (webhookDraftErrors(draft).length > 0) return null;
152
const maxDeliveryAgeMinutes = parseMaxDeliveryAge(draft.maxDeliveryAgeMinutes);
153
+ const replayKey = draft.replayKey.trim();
154
+ const coalesceKey = draft.coalesceKey.trim();
155
+ const coalesceWindowSeconds = draft.coalesceWindowSeconds.trim()
156
+ ? Number(draft.coalesceWindowSeconds)
157
+ : undefined;
158
if (maxDeliveryAgeMinutes === undefined) return null;
159
const parsedFilters = parseWebhookFilterDrafts(draft.filters);
160
if (!("filters" in parsedFilters)) return null;
@@ -112,6 +170,15 @@ export function scheduleFromDraft(draft: DraftState): ScheduledTaskUpsertSchedul
170
}
171
: null,
172
maxDeliveryAgeMinutes,
173
+ ...(replayKey || coalesceKey || coalesceWindowSeconds !== undefined
174
+ ? {
175
+ dedupe: {
176
+ ...(replayKey ? { replayKey } : {}),
177
+ ...(coalesceKey ? { coalesceKey } : {}),
178
+ ...(coalesceWindowSeconds === undefined ? {} : { coalesceWindowSeconds }),
179
+ },
180
+ }
181
+ : { dedupe: {} }),
182
// Only edited filters are sent; the server keeps the stored ones otherwise.
183
...(webhookFilterDraftsChanged(draft.savedFilters, draft.filters)
184
? { filters: parsedFilters.filters }
@@ -173,6 +240,12 @@ export function taskToDraft(task: ScheduledTask): DraftState {
240
}
241
: { signatureEnabled: false, ...WEBHOOK_SIGNATURE_DEFAULTS }),
242
signatureSecret: "",
243
+ replayKey: schedule.type === "webhook" ? (schedule.dedupe?.replayKey ?? "") : "",
244
+ coalesceKey: schedule.type === "webhook" ? (schedule.dedupe?.coalesceKey ?? "") : "",
245
+ coalesceWindowSeconds:
246
+ schedule.type === "webhook" && schedule.dedupe?.coalesceWindowSeconds !== undefined
247
+ ? String(schedule.dedupe.coalesceWindowSeconds)
248
+ : "",
249
maxDeliveryAgeMinutes:
250
schedule.type === "webhook" && schedule.maxDeliveryAgeMinutes != null
251
? String(schedule.maxDeliveryAgeMinutes)
docs/user/project-settings.md
+14
@@ -118,6 +118,20 @@ compare as text, and a value that is an object or a list only passes
118
**exists**. Separate **is one of** values with commas; a value can't contain
119
a comma or start or end with a space.
120
121
+To suppress repeated deliveries, set a **Replay key** template, such as
122
+`{{headers.delivery-id}}`. The same key is ignored for seven days per task.
123
+To combine a burst, set a **Coalesce key**, such as `{{body.id}}`, and a window
124
+of 10–3600 seconds (120 by default). One run uses the latest body, headers,
125
+and query when the window from the first delivery closes; later requests
126
+do not extend it. Pending windows survive a server restart. Keys must contain
127
+request placeholders; if any placeholder is missing, or the render is empty,
128
+that key's rule is skipped. Clear the templates to turn dedupe off.
129
+
130
+Duplicate and coalesced log entries share a limit of 10 per task per minute.
131
+Additional deliveries are accepted without a log entry, and coalesced deliveries
132
+always replace the persisted payload. Coalesced updates count toward a separate
133
+60-per-minute validation limit; exceeding it receives HTTP 429.
134
+
135
On desktop and web, pick **Deliveries** from a task's menu to see recent
136
requests and the prompt each one produced.
137
packages/contracts/src/scheduledTask.test.ts
+47
-4
@@ -2,14 +2,49 @@ import { describe, expect, it } from "vite-plus/test";
2
import * as Schema from "effect/Schema";
3
4
import {
5
+ isWebhookDedupeTemplate,
6
MIN_SCHEDULED_TASK_INTERVAL_MS,
7
ScheduledTaskListWebhookDeliveriesResult,
8
ScheduledTaskSchedule,
9
ScheduledTaskUpsertSchedule,
10
+ ScheduledTaskWebhookDeliverySummary,
11
} from "./scheduledTask.ts";
12
13
const decodeSchedule = Schema.decodeUnknownSync(ScheduledTaskSchedule);
14
const decodeUpsertSchedule = Schema.decodeUnknownSync(ScheduledTaskUpsertSchedule);
15
+const decodeDeliverySummary = Schema.decodeSync(ScheduledTaskWebhookDeliverySummary);
16
+const decodeDeliveryList = Schema.decodeUnknownSync(ScheduledTaskListWebhookDeliveriesResult);
17
+
18
+it("rejects a long unterminated dotted dedupe key without excessive backtracking", () => {
19
+ const template = "{{body." + "a.".repeat(5000) + "x";
20
+ const startedAt = performance.now();
21
+ const valid = isWebhookDedupeTemplate(template);
22
+ const elapsedMs = performance.now() - startedAt;
23
+ expect(valid).toBe(false);
24
+ expect(elapsedMs).toBeLessThan(50);
25
+ for (const field of ["replayKey", "coalesceKey"]) {
26
+ expect(() =>
27
+ decodeUpsertSchedule({ type: "webhook", dedupe: { [field]: template } }),
28
+ ).toThrow();
29
+ }
30
+});
31
+
32
+it.each(["duplicate", "coalesced", "future_outcome"])("decodes delivery outcome %s", (outcome) => {
33
+ expect(
34
+ decodeDeliverySummary({
35
+ id: "delivery:test",
36
+ taskId: "scheduled-task:test",
37
+ receivedAt: "2026-10-20T00:00:00.000Z",
38
+ method: "POST",
39
+ contentType: null,
40
+ bodyBytes: 0,
41
+ outcome,
42
+ signatureVerified: false,
43
+ missingFields: [],
44
+ error: null,
45
+ }).outcome,
46
+ ).toBe(outcome);
47
+});
48
49
describe("ScheduledTaskSchedule", () => {
50
it("keeps legacy sub-minute persisted schedules readable", () => {
@@ -52,6 +87,15 @@ describe("ScheduledTaskUpsertSchedule", () => {
87
});
88
});
89
90
+it.each(["literal", " ", "\t\n", "{{body.bad", "{{unknown.field}}"])(
91
+ "rejects invalid dedupe template %s",
92
+ (replayKey) => {
93
+ expect(() => decodeUpsertSchedule({ type: "webhook", dedupe: { replayKey } })).toThrow();
94
+ expect(() =>
95
+ decodeUpsertSchedule({ type: "webhook", dedupe: { coalesceKey: replayKey } }),
96
+ ).toThrow();
97
+ },
98
+);
99
describe("webhook filters", () => {
100
const accepts = (filters: unknown) => {
101
try {
@@ -114,9 +158,8 @@ describe("ScheduledTaskWebhookDeliverySummary", () => {
158
missingFields: [],
159
error: null,
160
};
117
- expect(
118
- Schema.decodeUnknownSync(ScheduledTaskListWebhookDeliveriesResult)({ deliveries: [summary] })
119
- .deliveries[0]?.outcome,
120
- ).toBe("some_future_outcome");
161
+ expect(decodeDeliveryList({ deliveries: [summary] }).deliveries[0]?.outcome).toBe(
162
+ "some_future_outcome",
163
+ );
164
});
165
});
packages/contracts/src/scheduledTask.ts
+26
@@ -83,6 +83,26 @@ const WebhookMaxDeliveryAgeMinutes = Schema.Int.check(
83
"Skip requests the relay held longer than this many minutes while the environment was offline. Null runs every request.",
84
});
85
86
+export function isWebhookDedupeTemplate(template: string): boolean {
87
+ const placeholders = /\{\{\s*(?:(?:body|headers|query)(?:\.[^{}.\s]+)*|request)\s*\}\}/g;
88
+ const matches = [...template.matchAll(placeholders)];
89
+ const remainder = template.replace(placeholders, "");
90
+ return matches.length > 0 && !remainder.includes("{{") && !remainder.includes("}}");
91
+}
92
+
93
+const WebhookDedupeTemplate = Schema.String.check(Schema.makeFilter(isWebhookDedupeTemplate));
94
+
95
+const ScheduledTaskWebhookDedupe = Schema.Struct({
96
+ replayKey: Schema.optional(WebhookDedupeTemplate),
97
+ coalesceKey: Schema.optional(WebhookDedupeTemplate),
98
+ coalesceWindowSeconds: Schema.optional(
99
+ Schema.Int.check(Schema.isBetween({ minimum: 10, maximum: 3600 })),
100
+ ),
101
+}).annotate({
102
+ description:
103
+ "Replay key template (7 days), coalesce key template, and fixed window in seconds (default 120).",
104
+});
105
+
106
export const MAX_WEBHOOK_FILTERS = 16;
107
108
const WebhookFilterPath = TrimmedNonEmptyString.check(
@@ -132,6 +152,7 @@ const ScheduledTaskWebhookSchedule = Schema.Struct({
152
}),
153
signature: Schema.NullOr(ScheduledTaskWebhookSignature),
154
maxDeliveryAgeMinutes: Schema.optional(Schema.NullOr(WebhookMaxDeliveryAgeMinutes)),
155
+ dedupe: Schema.optional(ScheduledTaskWebhookDedupe),
156
filters: Schema.optional(ScheduledTaskWebhookFilters),
157
}).annotate({
158
description:
@@ -159,6 +180,7 @@ const ScheduledTaskUpsertWebhookSchedule = Schema.Struct({
180
description: "Signature check; omit or null to accept requests by URL token only.",
181
}),
182
maxDeliveryAgeMinutes: Schema.optional(Schema.NullOr(WebhookMaxDeliveryAgeMinutes)),
183
+ dedupe: Schema.optional(ScheduledTaskWebhookDedupe),
184
filters: Schema.optional(ScheduledTaskWebhookFilters),
185
}).annotate({
186
description:
@@ -294,6 +316,8 @@ export type ScheduledTaskWebhookDeliveryId = typeof ScheduledTaskWebhookDelivery
316
export const ScheduledTaskWebhookDeliveryOutcome = Schema.Literals([
317
"accepted",
318
"dispatch_failed",
319
+ "duplicate",
320
+ "coalesced",
321
"rejected_signature",
322
"disabled",
323
"rate_limited",
@@ -312,6 +336,7 @@ export const ScheduledTaskWebhookDeliverySummary = Schema.Struct({
336
// Any string, so an outcome added later doesn't break a client's whole list.
337
outcome: Schema.Union([ScheduledTaskWebhookDeliveryOutcome, Schema.String]),
338
signatureVerified: Schema.Boolean,
339
+ pendingWindow: Schema.optional(Schema.Boolean),
340
missingFields: Schema.Array(Schema.String),
341
error: Schema.NullOr(Schema.String),
342
});
@@ -348,6 +373,7 @@ export type ScheduledTaskListWebhookDeliveriesResult =
373
export const ScheduledTaskGetWebhookDeliveryInput = Schema.Struct({
374
id: ScheduledTaskId,
375
deliveryId: ScheduledTaskWebhookDeliveryId,
376
+ acceptAnyOutcome: Schema.optional(Schema.Boolean),
377
});
378
export type ScheduledTaskGetWebhookDeliveryInput = typeof ScheduledTaskGetWebhookDeliveryInput.Type;
379