@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 2,200 lines 89.4 KB
Raw
1 // @effect-diagnostics-next-line nodeBuiltinImport:off -- Effect's Crypto has no createHmac.
2 import * as NodeCrypto from "node:crypto";
3
4 import * as NodePlatformCrypto from "@effect/platform-node/NodeCrypto";
5 import { assert, it } from "@effect/vitest";
6 import {
7 type ManagedPolicyDocument,
8 ScheduledTaskUpsertInput,
9 ScheduledTaskWebhookDeliveryId,
10 SecretRequestError,
11 } from "@t3tools/contracts";
12 import * as DateTime from "effect/DateTime";
13 import * as SecureRandom from "effect/crypto/SecureRandom";
14 import * as Deferred from "effect/Deferred";
15 import * as Effect from "effect/Effect";
16 import * as Exit from "effect/Exit";
17 import * as Fiber from "effect/Fiber";
18 import * as Layer from "effect/Layer";
19 import * as Queue from "effect/Queue";
20 import * as EffectScheduler from "effect/Scheduler";
21 import * as Schema from "effect/Schema";
22 import * as SqlClient from "effect/sql/SqlClient";
23 import * as TestClock from "effect/testing/TestClock";
24 import * as Tracer from "effect/Tracer";
25
26 import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts";
27 import * as ManagedPolicy from "../policy/ManagedPolicy.ts";
28 import * as ThreadManagementService from "../orchestration-v2/ThreadManagementService.ts";
29 import * as SecretRequests from "../secrets/SecretRequests.ts";
30 import * as ManagedProjectFolders from "../project/ManagedProjectFolders.ts";
31 import * as SqlitePersistence from "../persistence/Sqlite.ts";
32 import * as Scheduler from "../scheduling/Scheduler.ts";
33 import * as ScheduledTaskService from "./ScheduledTaskService.ts";
34 import { renderWebhookPrompt } from "./webhookTemplate.ts";
35
36 const decodeUpsertInput = Schema.decodeUnknownEffect(ScheduledTaskUpsertInput);
37
38 it.effect(
39 "preserves omitted dedupe settings, releases duplicate claims, and retains coalesced claims",
40 () =>
41 withService(({ service, sql, launches }) =>
42 Effect.gen(function* () {
43 const dedupe = {
44 replayKey: "{{headers.delivery-id}}",
45 coalesceKey: "{{body.group}}",
46 coalesceWindowSeconds: 10,
47 };
48 const { task } = yield* service.upsert(
49 yield* webhookTaskInput({ schedule: { type: "webhook", dedupe } }),
50 );
51 const updated = yield* service.upsert(yield* webhookTaskInput());
52 assert.deepEqual(
53 updated.task.schedule.type === "webhook" && updated.task.schedule.dedupe,
54 dedupe,
55 );
56 const request = (id: string, bodyText = "{}", relayDeliveryId?: string) =>
57 requestFor(task, {
58 headers: { "delivery-id": id },
59 bodyText,
60 body: new TextEncoder().encode(bodyText),
61 ...(relayDeliveryId === undefined ? {} : { relayDeliveryId }),
62 });
63 yield* service.triggerWebhook(request("A"));
64 yield* Queue.take(launches);
65 yield* service.triggerWebhook(request("B1", '{"group":"B"}'));
66 for (let i = 0; i < 5; i++) {
67 const duplicate = yield* service.triggerWebhook(request("A", "{}", `duplicate-${i}`));
68 assert.equal(duplicate._tag === "accepted" && duplicate.outcome, "duplicate");
69 const coalesced = yield* service.triggerWebhook(
70 request(`B${i + 2}`, '{"group":"B"}', `coalesced-${i}`),
71 );
72 assert.equal(coalesced._tag === "accepted" && coalesced.outcome, "coalesced");
73 }
74 const claims = yield* sql<{
75 relay_delivery_id: string;
76 }>`SELECT relay_delivery_id FROM scheduled_task_webhook_relay_deliveries ORDER BY relay_delivery_id`;
77 assert.deepEqual(
78 claims.map((claim) => claim.relay_delivery_id),
79 Array.from({ length: 5 }, (_, i) => `coalesced-${i}`),
80 );
81 assert.isTrue(
82 (yield* service.listWebhookDeliveries({ id: task.id })).deliveries.every(
83 (delivery) => delivery.outcome !== "duplicate" && delivery.outcome !== "coalesced",
84 ),
85 );
86 const fresh = yield* service.triggerWebhook(request("C", "{}", "duplicate-0"));
87 assert.equal(fresh._tag === "accepted" && fresh.outcome, "accepted");
88 yield* Queue.take(launches);
89 const cleared = yield* service.upsert(
90 yield* webhookTaskInput({ schedule: { type: "webhook", dedupe: {} } }),
91 );
92 assert.deepEqual(
93 cleared.task.schedule.type === "webhook" && cleared.task.schedule.dedupe,
94 {},
95 );
96 }),
97 ),
98 );
99
100 const waitForSpan = <A, E, R>(work: Effect.Effect<A, E, R>, name: string) =>
101 Effect.gen(function* () {
102 const done = yield* Deferred.make<void>();
103 const tracer = Tracer.make({
104 span: (options) => {
105 const span = new Tracer.NativeSpan(options);
106 const end = span.end.bind(span);
107 span.end = (...args) => {
108 end(...args);
109 if (span.name === name) Deferred.doneUnsafe(done, Effect.void);
110 };
111 return span;
112 },
113 });
114 const result = yield* work.pipe(Effect.withTracer(tracer));
115 yield* Deferred.await(done);
116 return result;
117 });
118
119 const drainPendingDispatch = (workers: Map<string, Effect.Effect<void>>) =>
120 waitForSpan(workers.get("webhook-coalescing")!, "ScheduledTaskService.dispatchPendingWebhook");
121
122 type LaunchInput = ThreadLaunchService.ThreadLaunchInput;
123
124 const webhookTaskInput = (overrides: Record<string, unknown> = {}) =>
125 decodeUpsertInput({
126 id: "scheduled-task:hook",
127 title: "Review PRs",
128 prompt: "Review this PR: {{body.pull_request.url}}",
129 enabled: true,
130 schedule: { type: "webhook" },
131 projectId: "project-webhook",
132 workspaceStrategy: { type: "root" },
133 modelSelection: { instanceId: "codex", model: "gpt-5.4" },
134 runtimeMode: "full-access",
135 interactionMode: "default",
136 ...overrides,
137 });
138
139 const pullRequestBody = new TextEncoder().encode(
140 JSON.stringify({ pull_request: { url: "https://github.com/org/repo/pull/45" } }),
141 );
142
143 const requestFor = (
144 task: { readonly id: string; readonly webhook?: { readonly path: string } | undefined },
145 overrides: Partial<ScheduledTaskService.WebhookTriggerRequest> = {},
146 ): ScheduledTaskService.WebhookTriggerRequest => ({
147 hookId: task.id,
148 token: task.webhook?.path.split("/").at(-1) ?? "",
149 method: "POST",
150 path: `/api/hooks/${task.id}`,
151 query: "",
152 headers: { "content-type": "application/json" },
153 body: pullRequestBody,
154 bodyText: new TextDecoder().decode(pullRequestBody),
155 ...overrides,
156 });
157
158 const withService = <A, E>(
159 body: (input: {
160 readonly service: ScheduledTaskService.ScheduledTaskService["Service"];
161 readonly launches: Queue.Queue<LaunchInput>;
162 readonly sql: SqlClient.SqlClient;
163 readonly secretsByRef: Map<string, string>;
164 }) => Effect.Effect<A, E, never>,
165 options: {
166 readonly onFrame?: () => void;
167 readonly frameGate?: {
168 readonly started: Deferred.Deferred<void>;
169 readonly release: Deferred.Deferred<void>;
170 };
171 readonly completedLaunches?: Set<string>;
172 readonly failLaunches?: { remaining: number };
173 readonly gate?: Deferred.Deferred<void>;
174 readonly gateForTask?: (input: LaunchInput) => Deferred.Deferred<void> | undefined;
175 readonly relayHookBaseUrl?: string;
176 readonly sql?: SqlClient.SqlClient;
177 readonly workers?: Map<string, Effect.Effect<void>>;
178 readonly policy?: ManagedPolicyDocument;
179 } = {},
180 ) =>
181 Effect.gen(function* () {
182 const launches = yield* Queue.unbounded<LaunchInput>();
183 const secretsByRef = new Map<string, string>();
184 const layerDependencies = Layer.mergeAll(
185 NodePlatformCrypto.layerDigest,
186 NodePlatformCrypto.layerMac,
187 options.frameGate === undefined && options.onFrame === undefined
188 ? NodePlatformCrypto.layer
189 : Layer.effect(
190 SecureRandom.SecureRandom,
191 Effect.gen(function* () {
192 const crypto = yield* SecureRandom.SecureRandom;
193 return {
194 ...crypto,
195 randomBytes: (size: number) =>
196 Effect.suspend(() => {
197 if (size === 6) options.onFrame?.();
198 return size === 6 && options.frameGate !== undefined
199 ? Deferred.succeed(options.frameGate.started, undefined).pipe(
200 Effect.andThen(Deferred.await(options.frameGate.release)),
201 Effect.andThen(crypto.randomBytes(size)),
202 )
203 : crypto.randomBytes(size);
204 }),
205 };
206 }),
207 ).pipe(Layer.provide(NodePlatformCrypto.layer)),
208 options.workers === undefined
209 ? Scheduler.layer
210 : Layer.succeed(Scheduler.Scheduler, {
211 register: <E, R>(name: string, work: Effect.Effect<void, E, R>) =>
212 Effect.gen(function* () {
213 const context = yield* Effect.context<R>();
214 options.workers!.set(name, work.pipe(Effect.provideContext(context), Effect.orDie));
215 }),
216 }),
217 Layer.mock(ThreadLaunchService.ThreadLaunchService)({
218 launch: (input) =>
219 Effect.suspend(() => {
220 if (options.completedLaunches?.has(input.commandId))
221 return Effect.succeed({ threadId: "thread-1", resumed: false } as never);
222 options.completedLaunches?.add(input.commandId);
223 return Queue.offer(launches, input).pipe(
224 Effect.andThen(
225 (options.gateForTask?.(input) ?? options.gate)
226 ? Deferred.await((options.gateForTask?.(input) ?? options.gate)!)
227 : Effect.void,
228 ),
229 Effect.andThen(
230 Effect.sync(() => {
231 if (options.failLaunches && options.failLaunches.remaining-- > 0)
232 throw new Error("injected start failure");
233 }),
234 ),
235 Effect.as({ threadId: "thread-1", resumed: false } as never),
236 );
237 }),
238 }),
239 Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({
240 namedProjectsRoot: "/projects",
241 isScratchProject: () => Effect.succeed(false),
242 }),
243 Layer.mock(ThreadManagementService.ThreadManagementService)({}),
244 Layer.mock(SecretRequests.SecretRequests)({
245 consume: ({ ref }) => {
246 const value = secretsByRef.get(ref);
247 secretsByRef.delete(ref);
248 return value === undefined
249 ? Effect.fail(new SecretRequestError({ reason: "ref_unavailable" }))
250 : Effect.succeed(value);
251 },
252 }),
253 Layer.succeed(
254 ScheduledTaskService.ScheduledTaskWebhookOrigin,
255 Effect.succeed({ relayHookBaseUrl: options.relayHookBaseUrl ?? null }),
256 ),
257 ManagedPolicy.layerTest(options.policy),
258 );
259 return yield* Effect.gen(function* () {
260 const service = yield* ScheduledTaskService.ScheduledTaskService;
261 const sql = yield* SqlClient.SqlClient;
262 return yield* body({ service, launches, sql, secretsByRef });
263 }).pipe(
264 Effect.provide(
265 Layer.fresh(ScheduledTaskService.layer.pipe(Layer.provide(layerDependencies))),
266 ),
267 );
268 }).pipe(
269 Effect.provide(
270 options.sql === undefined
271 ? SqlitePersistence.layerMemory
272 : Layer.succeed(SqlClient.SqlClient, options.sql),
273 ),
274 );
275
276 it.effect("dispatches exactly the rendered prompt and logs the delivery", () =>
277 withService(({ service, launches }) =>
278 Effect.gen(function* () {
279 const { task } = yield* service.upsert(yield* webhookTaskInput());
280 assert.equal(task.nextRunAt, null);
281 assert.isDefined(task.webhook);
282 assert.isTrue(task.webhook!.path.startsWith("/api/hooks/scheduled-task%3Ahook/"));
283 assert.equal(task.webhook!.url, null);
284
285 const result = yield* service.triggerWebhook(requestFor(task));
286 assert.equal(result._tag, "accepted");
287 const launched = yield* Queue.take(launches);
288 const framed =
289 /^Review this PR: <(webhook-data-[0-9a-f]{12})>https:\/\/github\.com\/org\/repo\/pull\/45<\/\1>\n\nText between <\1> and <\/\1> was copied from the webhook request\./;
290 const frame = framed.exec(launched.initialMessage?.text ?? "")?.[1];
291 assert.isDefined(frame);
292 assert.equal(
293 launched.commandId,
294 `scheduled-task:${task.id}:webhook:${result._tag === "accepted" ? result.deliveryId : ""}`,
295 );
296
297 const { deliveries } = yield* service.listWebhookDeliveries({
298 id: task.id,
299 acceptAnyOutcome: true,
300 });
301 assert.equal(deliveries.length, 1);
302 assert.equal(deliveries[0]?.outcome, "accepted");
303 const { delivery } = yield* service.getWebhookDelivery({
304 id: task.id,
305 deliveryId: deliveries[0]!.id,
306 });
307 assert.equal(delivery.body, new TextDecoder().decode(pullRequestBody));
308 assert.equal(delivery.renderedPrompt, launched.initialMessage?.text);
309
310 yield* service.triggerWebhook(requestFor(task));
311 const next = framed.exec((yield* Queue.take(launches)).initialMessage?.text ?? "")?.[1];
312 assert.isDefined(next);
313 assert.notEqual(next, frame);
314 }),
315 ),
316 );
317
318 it("builds the relay hook URL from the managed tunnel's key, never the environment id", () => {
319 const relayUrl = "https://relay.example.com/";
320 assert.equal(
321 ScheduledTaskService.relayHookBaseUrl({
322 relayUrl,
323 tunnelName: "t3coderelay-managedendpoint-dev-julius-0123456789abcdef",
324 }),
325 "https://relay.example.com/v1/hooks/0123456789abcdef",
326 );
327 for (const tunnelName of [undefined, "t3coderelay-managedendpoint", "x-0123456789ABCDEF"]) {
328 assert.isNull(ScheduledTaskService.relayHookBaseUrl({ relayUrl, tunnelName }));
329 }
330 });
331
332 it.effect("gives webhook tasks a relay URL when the environment has a managed tunnel", () =>
333 withService(
334 ({ service }) =>
335 Effect.gen(function* () {
336 const { task } = yield* service.upsert(yield* webhookTaskInput());
337 const token = task.webhook!.path.split("/").at(-1);
338 assert.equal(
339 task.webhook?.url,
340 `https://relay.example.com/v1/hooks/0123456789abcdef/scheduled-task%3Ahook/${token}`,
341 );
342 }),
343 { relayHookBaseUrl: "https://relay.example.com/v1/hooks/0123456789abcdef" },
344 ),
345 );
346
347 it.effect("answers not found for a wrong token or unknown hook without logging", () =>
348 withService(({ service }) =>
349 Effect.gen(function* () {
350 const { task } = yield* service.upsert(yield* webhookTaskInput());
351 const wrongToken = yield* service.triggerWebhook(requestFor(task, { token: "nope" }));
352 assert.equal(wrongToken._tag, "not_found");
353 const unknown = yield* service.triggerWebhook(requestFor(task, { hookId: "missing" }));
354 assert.equal(unknown._tag, "not_found");
355 assert.equal(
356 (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true })).deliveries
357 .length,
358 0,
359 );
360 }),
361 ),
362 );
363
364 it.effect("rotating the token retires the old URL and saving keeps it", () =>
365 withService(({ service }) =>
366 Effect.gen(function* () {
367 const input = yield* webhookTaskInput();
368 const { task } = yield* service.upsert(input);
369 const saved = yield* service.upsert(yield* webhookTaskInput({ title: "Renamed" }));
370 assert.equal(saved.task.webhook?.path, task.webhook?.path);
371
372 const rotated = yield* service.rotateWebhookToken({ id: task.id });
373 assert.notEqual(rotated.task.webhook?.path, task.webhook?.path);
374 assert.equal((yield* service.triggerWebhook(requestFor(task)))._tag, "not_found");
375 assert.equal((yield* service.triggerWebhook(requestFor(rotated.task)))._tag, "accepted");
376 }),
377 ),
378 );
379
380 it.effect("checks the configured signature and keeps the secret write-only", () =>
381 withService(({ service }) =>
382 Effect.gen(function* () {
383 const { task } = yield* service.upsert(
384 yield* webhookTaskInput({
385 schedule: {
386 type: "webhook",
387 signature: {
388 header: "X-Hub-Signature-256",
389 encoding: "hex",
390 prefix: "sha256=",
391 secret: "s3cret",
392 },
393 },
394 }),
395 );
396 assert.deepEqual(task.schedule, {
397 type: "webhook",
398 signature: { header: "x-hub-signature-256", encoding: "hex", prefix: "sha256=" },
399 maxDeliveryAgeMinutes: null,
400 });
401 assert.isTrue(task.webhook?.hasSecret);
402
403 const unsigned = yield* service.triggerWebhook(requestFor(task));
404 assert.equal(unsigned._tag, "rejected_signature");
405
406 const signature = `sha256=${NodeCrypto.createHmac("sha256", "s3cret").update(pullRequestBody).digest("hex")}`;
407 const signed = yield* service.triggerWebhook(
408 requestFor(task, {
409 headers: { "content-type": "application/json", "x-hub-signature-256": signature },
410 }),
411 );
412 assert.equal(signed._tag, "accepted");
413
414 const resaved = yield* service.upsert(
415 yield* webhookTaskInput({
416 schedule: {
417 type: "webhook",
418 signature: { header: "x-hub-signature-256", encoding: "hex", prefix: "sha256=" },
419 },
420 }),
421 );
422 assert.isTrue(resaved.task.webhook?.hasSecret);
423
424 const outcomes = (yield* service.listWebhookDeliveries({
425 id: task.id,
426 acceptAnyOutcome: true,
427 })).deliveries.map((delivery) => [delivery.outcome, delivery.signatureVerified]);
428 assert.deepEqual(outcomes.toSorted(), [
429 ["accepted", true],
430 ["rejected_signature", false],
431 ]);
432 }),
433 ),
434 );
435
436 it.effect("a policy without webhook tasks keeps existing ones but refuses saves and requests", () =>
437 withService(({ service, sql }) =>
438 Effect.gen(function* () {
439 const { task } = yield* service.upsert(yield* webhookTaskInput());
440 const timer = yield* webhookTaskInput({
441 id: "scheduled-task:timer",
442 schedule: { type: "interval", everyMs: 3_600_000 },
443 });
444 yield* withService(
445 ({ service, launches }) =>
446 Effect.gen(function* () {
447 const blocked = "Example Corp's policy does not allow webhook tasks.";
448 const listed = (yield* service.list()).tasks.find(({ id }) => id === task.id);
449 assert.equal(listed?.blockedByPolicy, blocked);
450 assert.isTrue(listed?.enabled);
451
452 const refused = yield* service.upsert(yield* webhookTaskInput()).pipe(Effect.flip);
453 assert.equal(refused.message, blocked);
454 const created = yield* service
455 .upsert(yield* webhookTaskInput({ id: "scheduled-task:new-hook" }))
456 .pipe(Effect.flip);
457 assert.equal(created.message, blocked);
458
459 // Turned off, it can still be edited, but not turned back on.
460 yield* service.setEnabled({ id: task.id, enabled: false });
461 yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
462 const reenabled = yield* service
463 .setEnabled({ id: task.id, enabled: true })
464 .pipe(Effect.flip);
465 assert.equal(reenabled.message, blocked);
466
467 const result = yield* service.triggerWebhook(requestFor(task));
468 assert.equal(result._tag, "blocked_by_policy");
469 assert.equal(yield* Queue.size(launches), 0);
470 assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_deliveries`).length, 0);
471
472 // Timer tasks are unaffected, including a webhook task switched to a timer.
473 assert.isUndefined((yield* service.upsert(timer)).task.blockedByPolicy);
474 const switched = yield* service.upsert(
475 yield* webhookTaskInput({ schedule: { type: "interval", everyMs: 3_600_000 } }),
476 );
477 assert.isUndefined(switched.task.blockedByPolicy);
478 }),
479 {
480 sql,
481 policy: {
482 schemaVersion: 1,
483 organization: "Example Corp",
484 limits: { webhookTasks: false },
485 },
486 },
487 );
488 }),
489 ),
490 );
491
492 it.effect("logs but does not run deliveries to a disabled task, and refuses run now", () =>
493 withService(({ service, launches }) =>
494 Effect.gen(function* () {
495 const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
496 assert.equal((yield* service.triggerWebhook(requestFor(task)))._tag, "disabled");
497 assert.equal(yield* Queue.size(launches), 0);
498 const runNow = yield* service.runNow({ id: task.id }).pipe(Effect.flip);
499 assert.equal(runNow.message, "Webhook tasks run when their URL receives a request.");
500 }),
501 ),
502 );
503
504 it.effect("a save carrying a stale token cannot undo a rotation", () =>
505 withService(({ service }) =>
506 Effect.gen(function* () {
507 const { task } = yield* service.upsert(yield* webhookTaskInput());
508 const rotated = yield* service.rotateWebhookToken({ id: task.id });
509 const saved = yield* service.upsert(yield* webhookTaskInput({ title: "Edited" }));
510 assert.equal(saved.task.webhook?.path, rotated.task.webhook?.path);
511 assert.equal((yield* service.triggerWebhook(requestFor(task)))._tag, "not_found");
512 }),
513 ),
514 );
515
516 it.effect("keeps credential headers and query values out of the delivery log", () =>
517 withService(({ service }) =>
518 Effect.gen(function* () {
519 const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
520 yield* service.triggerWebhook(
521 requestFor(task, {
522 query: "page=2&api_key=k",
523 headers: {
524 "content-type": "application/json",
525 authorization: "Bearer sender-token",
526 "x-webhook-key": "k",
527 "x-github-event": "push",
528 },
529 }),
530 );
531 const [summary] = (yield* service.listWebhookDeliveries({
532 id: task.id,
533 acceptAnyOutcome: true,
534 })).deliveries;
535 const { delivery } = yield* service.getWebhookDelivery({
536 id: task.id,
537 deliveryId: summary!.id,
538 });
539 assert.equal(delivery.query, "page=2&api_key=[redacted]");
540 assert.equal(delivery.headers.authorization, "[redacted]");
541 assert.equal(delivery.headers["x-webhook-key"], "[redacted]");
542 assert.equal(delivery.headers["x-github-event"], "push");
543 }),
544 ),
545 );
546
547 it.effect("drops a character split by the delivery log's body limit", () =>
548 withService(({ service }) =>
549 Effect.gen(function* () {
550 const { task } = yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
551 const limit = 64 * 1024;
552 const euro = new TextEncoder().encode("€");
553 const body = new Uint8Array(limit - 1 + euro.length).fill(0x61);
554 body.set(euro, limit - 1);
555 yield* service.triggerWebhook(
556 requestFor(task, { body, bodyText: new TextDecoder().decode(body) }),
557 );
558 const [summary] = (yield* service.listWebhookDeliveries({
559 id: task.id,
560 acceptAnyOutcome: true,
561 })).deliveries;
562 const { delivery } = yield* service.getWebhookDelivery({
563 id: task.id,
564 deliveryId: summary!.id,
565 });
566 assert.isTrue(delivery.bodyTruncated);
567 assert.equal(delivery.bodyBytes, body.byteLength);
568 assert.equal(delivery.body.slice(-3), "aaa");
569 assert.equal(delivery.body.length, limit - 1);
570 }),
571 ),
572 );
573
574 const queuedDeliveryCases = [
575 {
576 change: "paused",
577 reason: "The task was paused before this delivery ran.",
578 apply: (service: ScheduledTaskService.ScheduledTaskService["Service"], id: string) =>
579 service.setEnabled({ id: id as never, enabled: false }),
580 },
581 {
582 change: "switched to an interval trigger",
583 reason: "The task's trigger changed before this delivery ran.",
584 apply: (service: ScheduledTaskService.ScheduledTaskService["Service"]) =>
585 webhookTaskInput({ schedule: { type: "interval", everyMs: 3_600_000 } }).pipe(
586 Effect.flatMap(service.upsert),
587 ),
588 },
589 ] as const;
590
591 it.effect.each(queuedDeliveryCases)(
592 "a delivery queued behind a run does not start once the task is $change",
593 ({ reason, apply }) =>
594 Effect.gen(function* () {
595 const gate = yield* Deferred.make<void>();
596 const deliveryReleased = yield* Deferred.make<void>();
597 const tracer = Tracer.make({
598 span: (options) => {
599 const span = new Tracer.NativeSpan(options);
600 const end = span.end.bind(span);
601 span.end = (...args) => {
602 end(...args);
603 if (span.name === "ScheduledTaskService.releaseFailedDelivery") {
604 Deferred.doneUnsafe(deliveryReleased, Effect.void);
605 }
606 };
607 return span;
608 },
609 });
610 yield* withService(
611 ({ service, launches }) =>
612 Effect.gen(function* () {
613 const { task } = yield* service.upsert(yield* webhookTaskInput());
614 yield* service.triggerWebhook(requestFor(task));
615 const queued = yield* service.triggerWebhook(requestFor(task));
616 yield* Queue.take(launches);
617 yield* apply(service, task.id);
618 yield* Deferred.succeed(gate, undefined);
619 const deliveryId = queued._tag === "accepted" ? queued.deliveryId : undefined;
620 yield* Deferred.await(deliveryReleased);
621 const delivery = (yield* service.getWebhookDelivery({
622 id: task.id,
623 deliveryId: deliveryId!,
624 })).delivery;
625 assert.equal(delivery.outcome, "dispatch_failed");
626 assert.equal(delivery.error, reason);
627 assert.equal(yield* Queue.size(launches), 0);
628 }),
629 { gate },
630 ).pipe(Effect.withTracer(tracer));
631 }),
632 );
633
634 it.effect("a save without a secret keeps a secret changed after it was read", () =>
635 withService(({ service }) =>
636 Effect.gen(function* () {
637 const signature = { header: "x-hub-signature-256", encoding: "hex", prefix: "sha256=" };
638 const { task } = yield* service.upsert(
639 yield* webhookTaskInput({
640 schedule: { type: "webhook", signature: { ...signature, secret: "old" } },
641 }),
642 );
643 yield* service.upsert(
644 yield* webhookTaskInput({
645 schedule: { type: "webhook", signature: { ...signature, secret: "new" } },
646 }),
647 );
648 yield* service.upsert(
649 yield* webhookTaskInput({ title: "Edited", schedule: { type: "webhook", signature } }),
650 );
651 const sign = (secret: string) =>
652 `sha256=${NodeCrypto.createHmac("sha256", secret).update(pullRequestBody).digest("hex")}`;
653 const withSignature = (secret: string) =>
654 requestFor(task, {
655 headers: { "content-type": "application/json", "x-hub-signature-256": sign(secret) },
656 });
657 assert.equal(
658 (yield* service.triggerWebhook(withSignature("old")))._tag,
659 "rejected_signature",
660 );
661 assert.equal((yield* service.triggerWebhook(withSignature("new")))._tag, "accepted");
662 }),
663 ),
664 );
665
666 it.effect("a held request already delivered directly runs only once", () =>
667 withService(({ service, launches }) =>
668 Effect.gen(function* () {
669 const { task } = yield* service.upsert(yield* webhookTaskInput());
670 const direct = yield* service.triggerWebhook(
671 requestFor(task, { relayDeliveryId: "relay-1" }),
672 );
673 const replayed = yield* service.triggerWebhook(
674 requestFor(task, { relayDeliveryId: "relay-1", receivedAt: "2026-10-04T10:00:00.000Z" }),
675 );
676 assert.equal(direct._tag, "accepted");
677 assert.deepEqual(replayed, { ...direct, outcome: "duplicate" } as typeof replayed);
678 yield* Queue.take(launches);
679 const logged = (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true }))
680 .deliveries;
681 assert.equal(logged.length, 1);
682 }),
683 ),
684 );
685
686 it.effect("relay retries recover failed acceptance without releasing another request's claim", () =>
687 withService(({ service, launches, sql }) =>
688 Effect.gen(function* () {
689 const { task } = yield* service.upsert(yield* webhookTaskInput());
690 const request = requestFor(task, { relayDeliveryId: "retry-after-log-failure" });
691 yield* sql`
692 CREATE TRIGGER fail_webhook_delivery_log
693 BEFORE INSERT ON scheduled_task_webhook_deliveries
694 BEGIN
695 SELECT RAISE(FAIL, 'injected delivery log failure');
696 END
697 `;
698 const error = yield* service.triggerWebhook(request).pipe(Effect.flip);
699 assert.equal(error.message, "Could not record webhook delivery.");
700 assert.equal(yield* Queue.size(launches), 0);
701 assert.isEmpty(
702 (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true })).deliveries,
703 );
704 yield* sql`DROP TRIGGER fail_webhook_delivery_log`;
705
706 const retried = yield* Effect.all(
707 [service.triggerWebhook(request), service.triggerWebhook(request)],
708 { concurrency: "unbounded" },
709 );
710 assert.deepEqual(
711 retried.map((result) => (result._tag === "accepted" ? result.outcome : result._tag)).sort(),
712 ["accepted", "duplicate"],
713 );
714 assert.include((yield* Queue.take(launches)).commandId, request.relayDeliveryId!);
715
716 yield* sql`
717 CREATE TRIGGER fail_webhook_claim
718 BEFORE INSERT ON scheduled_task_webhook_relay_deliveries
719 BEGIN
720 SELECT RAISE(FAIL, 'injected claim failure');
721 END
722 `;
723 const retryError = yield* service.triggerWebhook(request).pipe(Effect.flip);
724 assert.equal(retryError.message, "Could not record webhook delivery.");
725 yield* sql`DROP TRIGGER fail_webhook_claim`;
726 const replayed = yield* service.triggerWebhook(request);
727 assert.equal(replayed._tag === "accepted" ? replayed.outcome : replayed._tag, "duplicate");
728 assert.equal(yield* Queue.size(launches), 0);
729 assert.equal(
730 (yield* service.listWebhookDeliveries({ id: task.id, acceptAnyOutcome: true })).deliveries
731 .length,
732 1,
733 );
734 }),
735 ),
736 );
737
738 it.effect("a held request whose sender hung up mid-request still runs", () =>
739 withService(({ service, launches }) =>
740 Effect.gen(function* () {
741 const { task } = yield* service.upsert(yield* webhookTaskInput());
742 for (let step = 0; step < 60; step++) {
743 const request = requestFor(task, { relayDeliveryId: `hung-up-${step}` });
744 const fiber = yield* service
745 .triggerWebhook(request)
746 .pipe(Effect.provideService(EffectScheduler.MaxOpsBeforeYield, 8), Effect.forkChild);
747 for (let yields = 0; yields < step; yields++) yield* Effect.yieldNow;
748 yield* Fiber.interrupt(fiber);
749 const retried = yield* service.triggerWebhook(request);
750 assert.equal(retried._tag, "accepted");
751 const deliveryId = retried._tag === "accepted" ? retried.deliveryId : undefined;
752 yield* service.getWebhookDelivery({ id: task.id, deliveryId: deliveryId! });
753 const launch = yield* Queue.take(launches);
754 assert.include(launch.commandId, `hung-up-${step}`);
755 yield* TestClock.adjust("61 seconds");
756 }
757 assert.equal(yield* Queue.size(launches), 0);
758 }),
759 ),
760 );
761
762 it.effect("a held request runs once even after the log has trimmed it", () =>
763 withService(({ service, launches }) =>
764 Effect.gen(function* () {
765 const { task } = yield* service.upsert(yield* webhookTaskInput());
766 const first = yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "kept" }));
767 assert.equal(first._tag, "accepted");
768 yield* Queue.take(launches);
769 const paused = yield* service.upsert(yield* webhookTaskInput({ enabled: false }));
770 yield* Effect.forEach(Array.from({ length: 55 }), () =>
771 service.triggerWebhook(requestFor(paused.task)),
772 );
773 yield* service.upsert(yield* webhookTaskInput());
774 const replay = yield* service.triggerWebhook(requestFor(task, { relayDeliveryId: "kept" }));
775 assert.equal(replay._tag, "accepted");
776 assert.equal(yield* Queue.size(launches), 0);
777 }),
778 ),
779 );
780
781 it.effect("skips a held request older than the task's max age", () =>
782 withService(({ service, launches }) =>
783 Effect.gen(function* () {
784 const { task } = yield* service.upsert(
785 yield* webhookTaskInput({ schedule: { type: "webhook", maxDeliveryAgeMinutes: 30 } }),
786 );
787 const now = yield* DateTime.now;
788 const old = DateTime.formatIso(DateTime.subtract(now, { minutes: 31 }));
789 const fresh = DateTime.formatIso(DateTime.subtract(now, { minutes: 5 }));
790 const tooOld = yield* service.triggerWebhook(
791 requestFor(task, { relayDeliveryId: "old", receivedAt: old }),
792 );
793 assert.equal(tooOld._tag, "expired");
794 assert.equal(yield* Queue.size(launches), 0);
795 const ok = yield* service.triggerWebhook(
796 requestFor(task, { relayDeliveryId: "fresh", receivedAt: fresh }),
797 );
798 assert.equal(ok._tag, "accepted");
799 const outcomes = (yield* service.listWebhookDeliveries({
800 id: task.id,
801 acceptAnyOutcome: true,
802 })).deliveries.map((delivery) => delivery.outcome);
803 assert.includeMembers(outcomes, ["expired", "accepted"]);
804 }),
805 ),
806 );
807
808 const githubSignature = (secret: string) =>
809 `sha256=${NodeCrypto.createHmac("sha256", secret).update(pullRequestBody).digest("hex")}`;
810
811 it.effect("a signature can take the user's secret by ref, which works only once", () =>
812 withService(({ service, launches, secretsByRef }) =>
813 Effect.gen(function* () {
814 secretsByRef.set("secret-ref:00000000000000000000000000000001", "github-secret");
815 const githubSchedule = (secretRef: string) => ({
816 type: "webhook",
817 signature: { header: "x-hub-signature-256", encoding: "hex", prefix: "sha256=", secretRef },
818 });
819 const { task } = yield* service.upsert(
820 yield* webhookTaskInput({
821 schedule: githubSchedule("secret-ref:00000000000000000000000000000001"),
822 }),
823 );
824 assert.isTrue(task.webhook!.hasSecret);
825 const signed = yield* service.triggerWebhook(
826 requestFor(task, {
827 headers: {
828 "content-type": "application/json",
829 "x-hub-signature-256": githubSignature("github-secret"),
830 },
831 }),
832 );
833 assert.equal(signed._tag, "accepted");
834 yield* Queue.take(launches);
835
836 const reused = yield* service
837 .upsert(
838 yield* webhookTaskInput({
839 id: "scheduled-task:other",
840 schedule: githubSchedule("secret-ref:00000000000000000000000000000001"),
841 }),
842 )
843 .pipe(Effect.flip);
844 assert.include(reused.message, "already used");
845 }),
846 ),
847 );
848
849 it.effect("a retried save with an already used secretRef keeps the stored secret", () =>
850 withService(({ service, launches, secretsByRef }) =>
851 Effect.gen(function* () {
852 secretsByRef.set("secret-ref:00000000000000000000000000000002", "github-secret");
853 const save = webhookTaskInput({
854 id: undefined,
855 commandId: "command:mcp:schedule-task:release-hook",
856 schedule: {
857 type: "webhook",
858 signature: {
859 header: "x-hub-signature-256",
860 encoding: "hex",
861 prefix: "sha256=",
862 secretRef: "secret-ref:00000000000000000000000000000002",
863 },
864 },
865 });
866 const first = yield* service.upsert(yield* save);
867 const retried = yield* service.upsert(
868 yield* webhookTaskInput({
869 id: undefined,
870 commandId: "command:mcp:schedule-task:release-hook",
871 schedule: {
872 type: "webhook",
873 signature: {
874 header: "x-hub-signature-256",
875 encoding: "hex",
876 prefix: "sha256=",
877 secretRef: "secret-ref:00000000000000000000000000000002",
878 secret: "made-up-secret",
879 },
880 },
881 }),
882 );
883 assert.equal(retried.task.id, first.task.id);
884 const signed = yield* service.triggerWebhook(
885 requestFor(retried.task, {
886 headers: {
887 "content-type": "application/json",
888 "x-hub-signature-256": githubSignature("github-secret"),
889 },
890 }),
891 );
892 assert.equal(signed._tag, "accepted");
893 yield* Queue.take(launches);
894 }),
895 ),
896 );
897
898 it.effect("a signature without any secret is refused", () =>
899 withService(({ service }) =>
900 Effect.gen(function* () {
901 const failure = yield* service
902 .upsert(
903 yield* webhookTaskInput({
904 schedule: {
905 type: "webhook",
906 signature: { header: "x-hub-signature-256", encoding: "hex", prefix: "sha256=" },
907 },
908 }),
909 )
910 .pipe(Effect.flip);
911 assert.include(failure.message, "needs a signing secret");
912 }),
913 ),
914 );
915
916 it.effect("dedupes replay keys per task, persists them, and expires them after seven days", () =>
917 withService(
918 ({ service, launches, sql }) =>
919 Effect.gen(function* () {
920 const schedule = { type: "webhook", dedupe: { replayKey: "{{headers.delivery-id}}" } };
921 const { task } = yield* service.upsert(yield* webhookTaskInput({ schedule }));
922 const request = requestFor(task, { headers: { "delivery-id": "A" } });
923 const first = yield* service.triggerWebhook(request);
924 assert.equal(first._tag === "accepted" && first.outcome, "accepted");
925 yield* Queue.take(launches);
926 const replay = yield* service.triggerWebhook(request);
927 assert.equal(replay._tag === "accepted" && replay.outcome, "duplicate");
928 assert.equal(yield* Queue.size(launches), 0);
929 const other = yield* service.upsert(
930 yield* webhookTaskInput({ id: "scheduled-task:other", schedule }),
931 );
932 yield* service.triggerWebhook(requestFor(other.task, { headers: request.headers }));
933 yield* Queue.take(launches);
934 yield* withService(
935 ({ service: restarted }) =>
936 Effect.gen(function* () {
937 const replay = yield* restarted.triggerWebhook(request);
938 assert.equal(replay._tag === "accepted" && replay.outcome, "duplicate");
939 }),
940 { sql, workers: new Map() },
941 );
942 yield* TestClock.adjust("7 days");
943 const expired = yield* service.triggerWebhook(request);
944 assert.equal(expired._tag === "accepted" && expired.outcome, "accepted");
945 yield* Queue.take(launches);
946 assert.include(
947 (yield* service.listWebhookDeliveries({
948 id: task.id,
949 acceptAnyOutcome: true,
950 })).deliveries.map((d) => d.outcome),
951 "duplicate",
952 );
953 }),
954 { workers: new Map() },
955 ),
956 );
957
958 it.effect.each([1, 3])(
959 "coalesces %i deliveries into one run with the latest body, headers and query",
960 (count) => {
961 const workers = new Map<string, Effect.Effect<void>>();
962 return withService(
963 ({ service, launches }) =>
964 Effect.gen(function* () {
965 const { task } = yield* service.upsert(
966 yield* webhookTaskInput({
967 prompt: "{{body.value}} {{headers.version}} {{query.version}}",
968 schedule: {
969 type: "webhook",
970 dedupe: { coalesceKey: "{{body.key}}", coalesceWindowSeconds: 10 },
971 },
972 }),
973 );
974 for (let value = 1; value <= count; value++) {
975 const bodyText = JSON.stringify({ key: "B", value });
976 yield* service.triggerWebhook(
977 requestFor(task, {
978 bodyText,
979 body: new TextEncoder().encode(bodyText),
980 headers: { version: String(value) },
981 query: `version=${value}`,
982 }),
983 );
984 if (value < count) yield* TestClock.adjust("2 seconds");
985 }
986 const owner = (yield* service.listWebhookDeliveries({
987 id: task.id,
988 acceptAnyOutcome: true,
989 })).deliveries.find((delivery) => delivery.outcome === "accepted")!;
990 assert.isTrue(owner.pendingWindow);
991 const beforeDispatch = (yield* service.getWebhookDelivery({
992 id: task.id,
993 deliveryId: owner.id,
994 acceptAnyOutcome: true,
995 })).delivery;
996 assert.isTrue(beforeDispatch.pendingWindow);
997 assert.equal(beforeDispatch.body, JSON.stringify({ key: "B", value: count }));
998 assert.equal(beforeDispatch.headers.version, String(count));
999 assert.equal(beforeDispatch.query, `version=${count}`);
1000 assert.equal(
1001 beforeDispatch.bodyBytes,
1002 new TextEncoder().encode(beforeDispatch.body).byteLength,
1003 );
1004 assert.equal(yield* Queue.size(launches), 0);
1005 yield* workers.get("webhook-coalescing")!;
1006 assert.equal(yield* Queue.size(launches), 0);
1007 yield* TestClock.adjust(count === 1 ? "10 seconds" : "6 seconds");
1008 yield* workers.get("webhook-coalescing")!;
1009 const launched = yield* Queue.take(launches);
1010 assert.equal(
1011 (launched.initialMessage!.text.match(new RegExp(`>${count}<`, "g")) ?? []).length,
1012 3,
1013 );
1014 yield* workers.get("webhook-coalescing")!;
1015 assert.equal(yield* Queue.size(launches), 0);
1016 const outcomes = (yield* service.listWebhookDeliveries({
1017 id: task.id,
1018 acceptAnyOutcome: true,
1019 })).deliveries.map((d) => d.outcome);
1020 const afterDispatch = (yield* service.getWebhookDelivery({
1021 id: task.id,
1022 deliveryId: owner.id,
1023 acceptAnyOutcome: true,
1024 })).delivery;
1025 assert.isFalse(afterDispatch.pendingWindow);
1026 assert.equal(afterDispatch.body, beforeDispatch.body);
1027 assert.include(afterDispatch.renderedPrompt!, `>${count}<`);
1028 assert.equal(outcomes.filter((o) => o === "accepted").length, 1);
1029 assert.equal(outcomes.filter((o) => o === "coalesced").length, count - 1);
1030 }),
1031 { workers },
1032 );
1033 },
1034 );
1035
1036 it.effect("a coalesced window dispatches the prompt saved after it opened", () => {
1037 const workers = new Map<string, Effect.Effect<void>>();
1038 return withService(
1039 ({ service, launches }) =>
1040 Effect.gen(function* () {
1041 const schedule = {
1042 type: "webhook",
1043 dedupe: { coalesceKey: "{{body.id}}", coalesceWindowSeconds: 10 },
1044 };
1045 const { task } = yield* service.upsert(
1046 yield* webhookTaskInput({ prompt: "Use the original instructions.", schedule }),
1047 );
1048 const bodyText = JSON.stringify({ id: "review" });
1049 yield* service.triggerWebhook(
1050 requestFor(task, { bodyText, body: new TextEncoder().encode(bodyText) }),
1051 );
1052 yield* service.upsert(
1053 yield* webhookTaskInput({ prompt: "Use the updated instructions.", schedule }),
1054 );
1055 yield* TestClock.adjust("10 seconds");
1056 yield* workers.get("webhook-coalescing")!;
1057 const launched = yield* Queue.take(launches);
1058 assert.equal(launched.initialMessage?.text, "Use the updated instructions.");
1059 const owner = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries[0]!;
1060 const { delivery } = yield* service.getWebhookDelivery({
1061 id: task.id,
1062 deliveryId: owner.id,
1063 });
1064 assert.equal(delivery.renderedPrompt, "Use the updated instructions.");
1065 }),
1066 { workers },
1067 );
1068 });
1069
1070 it.effect("retrying a running window keeps the prompt it was dispatched with", () => {
1071 const workers = new Map<string, Effect.Effect<void>>();
1072 return withService(
1073 ({ service, sql, launches }) =>
1074 Effect.gen(function* () {
1075 const schedule = {
1076 type: "webhook",
1077 dedupe: { coalesceKey: "{{body}}", coalesceWindowSeconds: 10 },
1078 };
1079 const save = (prompt: string) =>
1080 Effect.flatMap(webhookTaskInput({ prompt, schedule }), service.upsert);
1081 const { task } = yield* save("Prompt when the window opened");
1082 yield* service.triggerWebhook(requestFor(task));
1083 yield* save("Prompt at dispatch");
1084 yield* sql`CREATE TRIGGER fail_completion BEFORE UPDATE ON scheduled_tasks
1085 WHEN NEW.last_run_status = 'succeeded' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`;
1086 yield* TestClock.adjust("10 seconds");
1087 yield* drainPendingDispatch(workers);
1088 const launched = yield* Queue.take(launches);
1089 assert.equal(launched.initialMessage?.text, "Prompt at dispatch");
1090 yield* save("Prompt saved after dispatch");
1091 yield* sql`DROP TRIGGER fail_completion`;
1092 yield* drainPendingDispatch(workers);
1093 assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0);
1094 const owner = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries[0]!;
1095 const { delivery } = yield* service.getWebhookDelivery({
1096 id: task.id,
1097 deliveryId: owner.id,
1098 });
1099 assert.equal(delivery.renderedPrompt, "Prompt at dispatch");
1100 }),
1101 { workers },
1102 );
1103 });
1104
1105 it.effect("a coalesced window fails its delivery when the task row cannot be read", () => {
1106 const workers = new Map<string, Effect.Effect<void>>();
1107 return withService(
1108 ({ service, sql, launches }) =>
1109 Effect.gen(function* () {
1110 const { task } = yield* service.upsert(
1111 yield* webhookTaskInput({
1112 schedule: {
1113 type: "webhook",
1114 dedupe: { coalesceKey: "{{body}}", coalesceWindowSeconds: 10 },
1115 },
1116 }),
1117 );
1118 yield* service.triggerWebhook(requestFor(task));
1119 yield* sql`UPDATE scheduled_tasks SET model_selection_json = 'not json' WHERE task_id = ${task.id}`;
1120 yield* TestClock.adjust("10 seconds");
1121 yield* drainPendingDispatch(workers);
1122 assert.equal(yield* Queue.size(launches), 0);
1123 assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0);
1124 const [delivery] = yield* sql<{
1125 outcome: string;
1126 error: string | null;
1127 }>`SELECT outcome, error FROM scheduled_task_webhook_deliveries`;
1128 assert.equal(delivery?.outcome, "dispatch_failed");
1129 // Failing before dispatch means a later successful read cannot send the stale snapshot.
1130 assert.equal(delivery?.error, "The pending webhook payload could not be prepared.");
1131 }),
1132 { workers },
1133 );
1134 });
1135
1136 it.effect("recovers a persisted coalescing window after restarting the service", () => {
1137 const workers = new Map<string, Effect.Effect<void>>();
1138 return withService(
1139 ({ service, sql }) =>
1140 Effect.gen(function* () {
1141 const { task } = yield* service.upsert(
1142 yield* webhookTaskInput({
1143 schedule: {
1144 type: "webhook",
1145 dedupe: { coalesceKey: "{{headers.content-type}}", coalesceWindowSeconds: 10 },
1146 },
1147 }),
1148 );
1149 yield* service.triggerWebhook(requestFor(task));
1150 yield* TestClock.adjust("4 seconds");
1151 const restartedWorkers = new Map<string, Effect.Effect<void>>();
1152 yield* withService(
1153 ({ launches }) =>
1154 Effect.gen(function* () {
1155 yield* restartedWorkers.get("webhook-coalescing")!;
1156 assert.equal(yield* Queue.size(launches), 0);
1157 yield* TestClock.adjust("6 seconds");
1158 yield* restartedWorkers.get("webhook-coalescing")!;
1159 yield* Queue.take(launches);
1160 yield* restartedWorkers.get("webhook-coalescing")!;
1161 assert.equal(yield* Queue.size(launches), 0);
1162 assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_pending`).length, 0);
1163 }),
1164 { sql, workers: restartedWorkers },
1165 );
1166 }),
1167 { workers },
1168 );
1169 });
1170
1171 it.effect("a pending dispatch interrupted at its permit does not block later polls", () => {
1172 const workers = new Map<string, Effect.Effect<void>>();
1173 // One shared dispatcher lets the test run the retry's forked worker to completion.
1174 const mixed = new EffectScheduler.MixedScheduler("sync");
1175 const dispatcher = mixed.makeDispatcher();
1176 return withService(
1177 ({ service, sql, launches }) =>
1178 Effect.gen(function* () {
1179 const { task } = yield* service.upsert(
1180 yield* webhookTaskInput({
1181 schedule: {
1182 type: "webhook",
1183 dedupe: { coalesceKey: "{{headers.content-type}}", coalesceWindowSeconds: 10 },
1184 },
1185 }),
1186 );
1187 yield* service.triggerWebhook(requestFor(task));
1188 yield* TestClock.adjust("10 seconds");
1189 const poll = workers.get("webhook-coalescing")!;
1190
1191 const entered = yield* Deferred.make<Fiber.Fiber<unknown, unknown>>();
1192 const interruptAtPermit = Tracer.make({
1193 span: (options) => {
1194 if (options.name === "ScheduledTaskService.webhookPermit") {
1195 const fiber = Fiber.getCurrent()!;
1196 Deferred.doneUnsafe(entered, Effect.succeed(fiber));
1197 fiber.interruptUnsafe();
1198 }
1199 return new Tracer.NativeSpan(options);
1200 },
1201 });
1202 const firstPoll = yield* poll.pipe(Effect.withTracer(interruptAtPermit), Effect.forkChild);
1203 assert.isTrue(Exit.hasInterrupts(yield* Fiber.await(yield* Deferred.await(entered))));
1204 yield* Fiber.await(firstPoll);
1205
1206 yield* poll;
1207 dispatcher.flush();
1208
1209 assert.equal(yield* Queue.size(launches), 1);
1210 assert.lengthOf(yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`, 0);
1211 }),
1212 { workers },
1213 ).pipe(
1214 Effect.provideService(EffectScheduler.Scheduler, {
1215 executionMode: mixed.executionMode,
1216 makeDispatcher: () => dispatcher,
1217 shouldYield: (fiber) => mixed.shouldYield(fiber),
1218 }),
1219 );
1220 });
1221
1222 it.effect("rolls back replay and pending state when recording a delivery fails", () => {
1223 const workers = new Map<string, Effect.Effect<void>>();
1224 return withService(
1225 ({ service, sql, launches }) =>
1226 Effect.gen(function* () {
1227 const { task } = yield* service.upsert(
1228 yield* webhookTaskInput({
1229 schedule: {
1230 type: "webhook",
1231 dedupe: {
1232 replayKey: "{{headers.content-type}}",
1233 coalesceKey: "{{headers.content-type}}",
1234 coalesceWindowSeconds: 10,
1235 },
1236 },
1237 }),
1238 );
1239 yield* sql`CREATE TRIGGER fail_dedupe_log BEFORE INSERT ON scheduled_task_webhook_deliveries BEGIN SELECT RAISE(FAIL, 'injected'); END`;
1240 yield* service.triggerWebhook(requestFor(task)).pipe(Effect.flip);
1241 assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_pending`).length, 0);
1242 assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_replay_keys`).length, 0);
1243 yield* sql`DROP TRIGGER fail_dedupe_log`;
1244 const retry = yield* service.triggerWebhook(requestFor(task));
1245 assert.equal(retry._tag === "accepted" && retry.outcome, "accepted");
1246 yield* TestClock.adjust("10 seconds");
1247 yield* workers.get("webhook-coalescing")!;
1248 yield* Queue.take(launches);
1249 }),
1250 { workers },
1251 );
1252 });
1253
1254 it.effect("queue-full deliveries release replay keys so a later retry runs", () =>
1255 Effect.gen(function* () {
1256 const gate = yield* Deferred.make<void>();
1257 yield* withService(
1258 ({ service, launches }) =>
1259 Effect.gen(function* () {
1260 const { task } = yield* service.upsert(
1261 yield* webhookTaskInput({
1262 schedule: { type: "webhook", dedupe: { replayKey: "{{headers.delivery-id}}" } },
1263 }),
1264 );
1265 const request = (i: number) =>
1266 requestFor(task, { headers: { "delivery-id": String(i) } });
1267 for (let i = 0; i < 20; i++) yield* service.triggerWebhook(request(i));
1268 yield* Queue.take(launches);
1269 const full = yield* service.triggerWebhook(request(20));
1270 assert.equal(full._tag === "rate_limited" && full.outcome, "queue_full");
1271 yield* Deferred.succeed(gate, undefined);
1272 for (let i = 1; i < 20; i++) yield* Queue.take(launches);
1273 const retry = yield* service.triggerWebhook(request(20));
1274 assert.equal(retry._tag === "accepted" && retry.outcome, "accepted");
1275 yield* Queue.take(launches);
1276 }),
1277 { gate },
1278 );
1279 }),
1280 );
1281
1282 it.effect.each(["task_json", "request_json"] as const)(
1283 "removes corrupt pending %s instead of retrying forever",
1284 (column) => {
1285 const workers = new Map<string, Effect.Effect<void>>();
1286 return withService(
1287 ({ service, sql }) =>
1288 Effect.gen(function* () {
1289 const { task } = yield* service.upsert(
1290 yield* webhookTaskInput({
1291 schedule: {
1292 type: "webhook",
1293 dedupe: {
1294 replayKey: "{{body}}",
1295 coalesceKey: "{{body}}",
1296 coalesceWindowSeconds: 10,
1297 },
1298 },
1299 }),
1300 );
1301 yield* service.triggerWebhook(requestFor(task));
1302 yield* sql`UPDATE scheduled_task_webhook_pending SET ${sql(column)} = 'broken'`;
1303 yield* TestClock.adjust("10 seconds");
1304 yield* drainPendingDispatch(workers);
1305 assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_pending`).length, 0);
1306 assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_replay_keys`).length, 0);
1307 assert.equal(
1308 (yield* service.listWebhookDeliveries({ id: task.id })).deliveries[0]!.outcome,
1309 "dispatch_failed",
1310 );
1311 yield* workers.get("webhook-coalescing")!;
1312 }),
1313 { workers },
1314 );
1315 },
1316 );
1317
1318 it.effect(
1319 "restarting after a coalesced launch before pending deletion does not launch or count it twice",
1320 () => {
1321 const workers = new Map<string, Effect.Effect<void>>();
1322 return withService(
1323 ({ service, sql, launches }) =>
1324 Effect.gen(function* () {
1325 const { task } = yield* service.upsert(
1326 yield* webhookTaskInput({
1327 schedule: {
1328 type: "webhook",
1329 dedupe: { coalesceKey: "{{body}}", coalesceWindowSeconds: 10 },
1330 },
1331 }),
1332 );
1333 const accepted = yield* service.triggerWebhook(requestFor(task));
1334 yield* sql`CREATE TRIGGER fail_pending_delete BEFORE DELETE ON scheduled_task_webhook_pending BEGIN SELECT RAISE(FAIL,'crash after launch'); END`;
1335 yield* TestClock.adjust("10 seconds");
1336 yield* drainPendingDispatch(workers);
1337 const launch = yield* Queue.take(launches);
1338 assert.equal(
1339 launch.commandId,
1340 `scheduled-task:${task.id}:webhook:${accepted._tag === "accepted" && accepted.deliveryId}`,
1341 );
1342 assert.equal((yield* service.list()).tasks[0]!.runCount, 1);
1343 assert.equal(
1344 (yield* sql`SELECT * FROM scheduled_task_webhook_pending WHERE state = 'completed'`)
1345 .length,
1346 1,
1347 );
1348 yield* sql`DROP TRIGGER fail_pending_delete`;
1349 const restartedWorkers = new Map<string, Effect.Effect<void>>();
1350 yield* withService(
1351 ({ service: restarted, launches: secondLaunches }) =>
1352 Effect.gen(function* () {
1353 yield* drainPendingDispatch(restartedWorkers);
1354 assert.equal(yield* Queue.size(secondLaunches), 0);
1355 assert.equal((yield* restarted.list()).tasks[0]!.runCount, 1);
1356 assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_pending`).length, 0);
1357 }),
1358 { sql, workers: restartedWorkers },
1359 );
1360 }),
1361 { workers },
1362 );
1363 },
1364 );
1365
1366 it.effect("guards bad-signature floods separately from valid run admission", () =>
1367 withService(({ service, sql }) =>
1368 Effect.gen(function* () {
1369 const { task } = yield* service.upsert(
1370 yield* webhookTaskInput({
1371 schedule: {
1372 type: "webhook",
1373 signature: { header: "signature", encoding: "hex", prefix: "", secret: "secret" },
1374 },
1375 }),
1376 );
1377 const results = yield* Effect.forEach(Array.from({ length: 75 }), () =>
1378 service.triggerWebhook(requestFor(task)),
1379 );
1380 assert.equal(results.filter((result) => result._tag === "rejected_signature").length, 60);
1381 assert.equal(results.filter((result) => result._tag === "rate_limited").length, 15);
1382 assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_deliveries`).length, 51);
1383 }),
1384 ),
1385 );
1386
1387 it.effect("start failures release both replay keys and relay claims for redelivery", () =>
1388 withService(
1389 ({ service, sql, launches }) =>
1390 Effect.gen(function* () {
1391 const { task } = yield* service.upsert(
1392 yield* webhookTaskInput({
1393 schedule: { type: "webhook", dedupe: { replayKey: "{{body}}" } },
1394 }),
1395 );
1396 const request = requestFor(task, { relayDeliveryId: "failed-start" });
1397 yield* waitForSpan(
1398 service.triggerWebhook(request),
1399 "ScheduledTaskService.runWebhookDelivery",
1400 );
1401 yield* Queue.take(launches);
1402 assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_replay_keys`).length, 0);
1403 assert.equal((yield* sql`SELECT * FROM scheduled_task_webhook_relay_deliveries`).length, 0);
1404 assert.equal((yield* service.list()).tasks[0]!.lastRunError, "injected start failure");
1405 const retry = yield* waitForSpan(
1406 service.triggerWebhook(request),
1407 "ScheduledTaskService.runWebhookDelivery",
1408 );
1409 assert.equal(retry._tag === "accepted" && retry.outcome, "accepted");
1410 yield* Queue.take(launches);
1411 assert.equal((yield* service.list()).tasks[0]!.lastRunStatus, "succeeded");
1412 }),
1413 { failLaunches: { remaining: 1 } },
1414 ),
1415 );
1416
1417 it.effect("a start failure releases the relay claim when the task row is replaced mid-run", () =>
1418 Effect.gen(function* () {
1419 const gate = yield* Deferred.make<void>();
1420 yield* withService(
1421 ({ service, sql, launches }) =>
1422 Effect.gen(function* () {
1423 const { task } = yield* service.upsert(yield* webhookTaskInput());
1424 const request = requestFor(task, { relayDeliveryId: "replaced-mid-run" });
1425 const first = yield* waitForSpan(
1426 service.triggerWebhook(request),
1427 "ScheduledTaskService.runWebhookDelivery",
1428 ).pipe(Effect.forkChild);
1429 yield* Queue.take(launches);
1430 // A same-id recreate while the launch is out: the stored row is no longer this run's.
1431 yield* sql`UPDATE scheduled_tasks SET last_run_status = 'never', last_run_at = NULL`;
1432 yield* Deferred.succeed(gate, undefined);
1433 yield* Fiber.join(first);
1434 const retry = yield* service.triggerWebhook(request);
1435 assert.equal(retry._tag === "accepted" && retry.outcome, "accepted");
1436 yield* Queue.take(launches);
1437 }),
1438 { gate, failLaunches: { remaining: 1 } },
1439 );
1440 }),
1441 );
1442
1443 it.effect("main retention keeps fifty runs and preserves filtered and dedupe buckets", () =>
1444 withService(({ service, sql, launches }) =>
1445 Effect.gen(function* () {
1446 const { task } = yield* service.upsert(yield* webhookTaskInput());
1447 yield* service.triggerWebhook(requestFor(task));
1448 yield* Queue.take(launches);
1449 for (const outcome of ["filtered", "duplicate"]) {
1450 yield* sql`WITH RECURSIVE n(value) AS (SELECT 1 UNION ALL SELECT value + 1 FROM n WHERE value < 20)
1451 INSERT INTO scheduled_task_webhook_deliveries
1452 SELECT ${outcome} || n.value, d.task_id, d.received_at, d.method, d.query, d.headers_json, d.body,
1453 d.body_bytes, d.body_truncated, ${outcome}, d.signature_verified, d.missing_fields_json, d.rendered_prompt, d.error
1454 FROM n CROSS JOIN (SELECT * FROM scheduled_task_webhook_deliveries WHERE outcome = 'accepted' LIMIT 1) d`;
1455 }
1456 for (let i = 0; i < 59; i++) {
1457 assert.equal((yield* service.triggerWebhook(requestFor(task)))._tag, "accepted");
1458 yield* Queue.take(launches);
1459 }
1460 const counts = yield* sql<{
1461 outcome: string;
1462 count: number;
1463 }>`SELECT outcome, COUNT(*) AS count FROM scheduled_task_webhook_deliveries GROUP BY outcome ORDER BY outcome`;
1464 assert.deepEqual(counts, [
1465 { outcome: "accepted", count: 50 },
1466 { outcome: "duplicate", count: 20 },
1467 { outcome: "filtered", count: 20 },
1468 ]);
1469 }),
1470 ),
1471 );
1472
1473 it.effect(
1474 "a valid request refunds its own reservation without expiring an interleaved rejection",
1475 () =>
1476 Effect.gen(function* () {
1477 const started = yield* Deferred.make<void>();
1478 const release = yield* Deferred.make<void>();
1479 yield* withService(
1480 ({ service, launches }) =>
1481 Effect.gen(function* () {
1482 const { task } = yield* service.upsert(
1483 yield* webhookTaskInput({
1484 schedule: {
1485 type: "webhook",
1486 signature: { header: "signature", encoding: "hex", prefix: "", secret: "secret" },
1487 },
1488 }),
1489 );
1490 const signature = NodeCrypto.createHmac("sha256", "secret")
1491 .update(pullRequestBody)
1492 .digest("hex");
1493 const valid = yield* service
1494 .triggerWebhook(requestFor(task, { headers: { signature } }))
1495 .pipe(Effect.forkChild);
1496 yield* Deferred.await(started);
1497 yield* TestClock.adjust("30 seconds");
1498 assert.equal(
1499 (yield* service.triggerWebhook(requestFor(task)))._tag,
1500 "rejected_signature",
1501 );
1502 yield* Deferred.succeed(release, undefined);
1503 assert.equal((yield* Fiber.join(valid))._tag, "accepted");
1504 yield* Queue.take(launches);
1505 // Only the valid reservation would have expired now; the rejection must still count.
1506 yield* TestClock.adjust("31 seconds");
1507 for (let i = 0; i < 59; i++)
1508 assert.equal(
1509 (yield* service.triggerWebhook(requestFor(task)))._tag,
1510 "rejected_signature",
1511 );
1512 assert.equal((yield* service.triggerWebhook(requestFor(task)))._tag, "rate_limited");
1513 }),
1514 { frameGate: { started, release } },
1515 );
1516 }),
1517 );
1518
1519 it.effect("keeps replay keys when dispatch succeeds but completion bookkeeping fails", () =>
1520 withService(({ service, sql, launches }) =>
1521 Effect.gen(function* () {
1522 const { task } = yield* service.upsert(
1523 yield* webhookTaskInput({
1524 schedule: {
1525 type: "webhook",
1526 dedupe: { replayKey: "{{body}}" },
1527 },
1528 }),
1529 );
1530 yield* sql`CREATE TRIGGER fail_completion BEFORE UPDATE ON scheduled_tasks
1531 WHEN NEW.last_run_status = 'succeeded' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`;
1532 yield* waitForSpan(
1533 service.triggerWebhook(requestFor(task)),
1534 "ScheduledTaskService.runWebhookDelivery",
1535 );
1536 yield* Queue.take(launches);
1537 assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`).length, 1);
1538 yield* sql`DROP TRIGGER fail_completion`;
1539 const retry = yield* service.triggerWebhook(requestFor(task));
1540 assert.equal(retry._tag === "accepted" && retry.outcome, "duplicate");
1541 assert.equal(yield* Queue.size(launches), 0);
1542 assert.equal((yield* service.list()).tasks[0]!.runCount, 1);
1543 }),
1544 ),
1545 );
1546
1547 it.effect.each([119_000, 120_000])(
1548 "bounds coalesced prompt logs and checks trimmed length for %i body characters",
1549 (length) => {
1550 const workers = new Map<string, Effect.Effect<void>>();
1551 return withService(
1552 ({ service, launches }) =>
1553 Effect.gen(function* () {
1554 const { task } = yield* service.upsert(
1555 yield* webhookTaskInput({
1556 prompt: `{{body.absent}}${" ".repeat(10_000)}{{body.text}}`,
1557 schedule: {
1558 type: "webhook",
1559 dedupe: { coalesceKey: "{{body.group}}", coalesceWindowSeconds: 10 },
1560 },
1561 }),
1562 );
1563 const bodyText = JSON.stringify({ group: "B", text: "x".repeat(length) });
1564 const accepted = yield* service.triggerWebhook(
1565 requestFor(task, { bodyText, body: new TextEncoder().encode(bodyText) }),
1566 );
1567 assert.equal(accepted._tag, "accepted");
1568 yield* TestClock.adjust("10 seconds");
1569 yield* drainPendingDispatch(workers);
1570 const delivery = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries[0]!;
1571 if (length === 119_000) {
1572 const launch = yield* Queue.take(launches);
1573 assert.isAbove(launch.initialMessage!.text.length, 120_000);
1574 assert.equal(delivery.outcome, "accepted");
1575 const detail = yield* service.getWebhookDelivery({
1576 id: task.id,
1577 deliveryId: delivery.id,
1578 });
1579 assert.equal(detail.delivery.renderedPrompt!.length, 64 * 1024);
1580 } else {
1581 assert.equal(yield* Queue.size(launches), 0);
1582 assert.equal(delivery.outcome, "dispatch_failed");
1583 assert.equal(delivery.error, "The filled-in prompt is too long.");
1584 }
1585 }),
1586 { workers },
1587 );
1588 },
1589 );
1590
1591 it.effect("oversized coalesced windows stay within the suppression bucket's limit", () => {
1592 const workers = new Map<string, Effect.Effect<void>>();
1593 return withService(
1594 ({ service, sql, launches }) =>
1595 Effect.gen(function* () {
1596 const { task } = yield* service.upsert(
1597 yield* webhookTaskInput({
1598 prompt: `{{body.absent}}${" ".repeat(10_000)}{{body.text}}`,
1599 schedule: {
1600 type: "webhook",
1601 dedupe: { coalesceKey: "{{body.group}}", coalesceWindowSeconds: 10 },
1602 },
1603 }),
1604 );
1605 const bodyText = JSON.stringify({ group: "B", text: "x".repeat(120_000) });
1606 for (let i = 0; i < 21; i++) {
1607 const accepted = yield* service.triggerWebhook(
1608 requestFor(task, { bodyText, body: new TextEncoder().encode(bodyText) }),
1609 );
1610 assert.equal(accepted._tag === "accepted" && accepted.outcome, "accepted");
1611 yield* TestClock.adjust("10 seconds");
1612 yield* drainPendingDispatch(workers);
1613 }
1614 assert.equal(yield* Queue.size(launches), 0);
1615 const counts = yield* sql<{
1616 outcome: string;
1617 count: number;
1618 }>`SELECT outcome, COUNT(*) AS count FROM scheduled_task_webhook_deliveries GROUP BY outcome`;
1619 assert.deepEqual(counts, [{ outcome: "dispatch_failed", count: 20 }]);
1620 }),
1621 { workers },
1622 );
1623 });
1624
1625 it.effect.each([1, 11])(
1626 "keeps relay claims for %i coalesced updates after their window closes",
1627 (count) => {
1628 const workers = new Map<string, Effect.Effect<void>>();
1629 return withService(
1630 ({ service, sql, launches }) =>
1631 Effect.gen(function* () {
1632 const { task } = yield* service.upsert(
1633 yield* webhookTaskInput({
1634 prompt: "{{body.value}}",
1635 schedule: {
1636 type: "webhook",
1637 dedupe: { coalesceKey: "{{body.group}}", coalesceWindowSeconds: 10 },
1638 },
1639 }),
1640 );
1641 const request = (value: number) => {
1642 const bodyText = JSON.stringify({ group: "B", value });
1643 return requestFor(task, {
1644 relayDeliveryId: `coalesced-relay-${value}`,
1645 bodyText,
1646 body: new TextEncoder().encode(bodyText),
1647 });
1648 };
1649 yield* service.triggerWebhook(request(0));
1650 for (let i = 1; i <= count; i++) {
1651 const result = yield* service.triggerWebhook(request(i));
1652 assert.equal(result._tag === "accepted" && result.outcome, "coalesced");
1653 }
1654 assert.equal(
1655 (yield* sql`SELECT 1 FROM scheduled_task_webhook_relay_deliveries`).length,
1656 count + 1,
1657 );
1658 yield* TestClock.adjust("10 seconds");
1659 yield* drainPendingDispatch(workers);
1660 assert.include((yield* Queue.take(launches)).initialMessage!.text, `>${count}<`);
1661 const retry = yield* service.triggerWebhook(request(count));
1662 assert.equal(retry._tag === "accepted" && retry.outcome, "duplicate");
1663 assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0);
1664 assert.equal((yield* service.list()).tasks[0]!.runCount, 1);
1665 assert.equal(yield* Queue.size(launches), 0);
1666 }),
1667 { workers },
1668 );
1669 },
1670 );
1671
1672 it.effect("failed windows release every member's replay key and relay claim", () => {
1673 const workers = new Map<string, Effect.Effect<void>>();
1674 return withService(
1675 ({ service, sql, launches }) =>
1676 Effect.gen(function* () {
1677 const { task } = yield* service.upsert(
1678 yield* webhookTaskInput({
1679 prompt: "{{body.value}}",
1680 schedule: {
1681 type: "webhook",
1682 dedupe: {
1683 replayKey: "{{headers.delivery-id}}",
1684 coalesceKey: "{{body.group}}",
1685 coalesceWindowSeconds: 10,
1686 },
1687 },
1688 }),
1689 );
1690 const request = (id: string) => {
1691 const bodyText = JSON.stringify({ group: "same", value: id });
1692 return requestFor(task, {
1693 bodyText,
1694 body: new TextEncoder().encode(bodyText),
1695 headers: { "delivery-id": id },
1696 relayDeliveryId: id,
1697 });
1698 };
1699 yield* service.triggerWebhook(request("A"));
1700 for (let i = 0; i < 12; i++) yield* service.triggerWebhook(request(`B${i}`));
1701 assert.equal(
1702 (yield* sql`SELECT 1 FROM scheduled_task_webhook_relay_deliveries`).length,
1703 13,
1704 );
1705 yield* TestClock.adjust("10 seconds");
1706 yield* drainPendingDispatch(workers);
1707 yield* Queue.take(launches);
1708 assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`).length, 0);
1709 assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_relay_deliveries`).length, 0);
1710 const retry = yield* service.triggerWebhook(request("B11"));
1711 assert.equal(retry._tag === "accepted" && retry.outcome, "accepted");
1712 yield* TestClock.adjust("10 seconds");
1713 yield* drainPendingDispatch(workers);
1714 assert.include((yield* Queue.take(launches)).initialMessage!.text, "B11");
1715 assert.equal((yield* service.list()).tasks[0]!.runCount, 2);
1716 }),
1717 { workers, failLaunches: { remaining: 1 } },
1718 );
1719 });
1720
1721 it.effect(
1722 "recovery finishes failed-window claim cleanup without counting the attempt twice",
1723 () => {
1724 const workers = new Map<string, Effect.Effect<void>>();
1725 return withService(
1726 ({ service, sql, launches }) =>
1727 Effect.gen(function* () {
1728 const { task } = yield* service.upsert(
1729 yield* webhookTaskInput({
1730 schedule: {
1731 type: "webhook",
1732 dedupe: {
1733 replayKey: "{{headers.delivery-id}}",
1734 coalesceKey: "{{body}}",
1735 coalesceWindowSeconds: 10,
1736 },
1737 },
1738 }),
1739 );
1740 const request = (id: string) =>
1741 requestFor(task, { headers: { "delivery-id": id }, relayDeliveryId: id });
1742 yield* service.triggerWebhook(request("A"));
1743 yield* service.triggerWebhook(request("B"));
1744 yield* sql`CREATE TRIGGER fail_claim_cleanup BEFORE DELETE ON scheduled_task_webhook_relay_deliveries
1745 BEGIN SELECT RAISE(FAIL, 'cleanup unavailable'); END`;
1746 yield* TestClock.adjust("10 seconds");
1747 yield* drainPendingDispatch(workers);
1748 yield* Queue.take(launches);
1749 assert.equal((yield* service.list()).tasks[0]!.runCount, 1);
1750 assert.equal(
1751 (yield* sql<{ state: string }>`SELECT state FROM scheduled_task_webhook_pending`)[0]!
1752 .state,
1753 "failed",
1754 );
1755 yield* sql`DROP TRIGGER fail_claim_cleanup`;
1756 const restartedWorkers = new Map<string, Effect.Effect<void>>();
1757 yield* withService(
1758 ({ service: restarted, launches: restartedLaunches }) =>
1759 Effect.gen(function* () {
1760 yield* drainPendingDispatch(restartedWorkers);
1761 assert.equal(yield* Queue.size(restartedLaunches), 0);
1762 assert.equal((yield* restarted.list()).tasks[0]!.runCount, 1);
1763 assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0);
1764 assert.equal(
1765 (yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`).length,
1766 0,
1767 );
1768 assert.equal(
1769 (yield* sql`SELECT 1 FROM scheduled_task_webhook_relay_deliveries`).length,
1770 0,
1771 );
1772 const retry = yield* restarted.triggerWebhook(request("B"));
1773 assert.equal(retry._tag === "accepted" && retry.outcome, "accepted");
1774 yield* TestClock.adjust("10 seconds");
1775 yield* drainPendingDispatch(restartedWorkers);
1776 yield* Queue.take(restartedLaunches);
1777 assert.equal((yield* restarted.list()).tasks[0]!.runCount, 2);
1778 }),
1779 { sql, workers: restartedWorkers },
1780 );
1781 }),
1782 { workers, failLaunches: { remaining: 1 } },
1783 );
1784 },
1785 );
1786
1787 it.effect("dispatches an overdue window exactly once after its deadline passed while stopped", () =>
1788 Effect.gen(function* () {
1789 const sql = yield* SqlClient.SqlClient;
1790 const task = yield* withService(
1791 ({ service, launches }) =>
1792 Effect.gen(function* () {
1793 const { task } = yield* service.upsert(
1794 yield* webhookTaskInput({
1795 schedule: {
1796 type: "webhook",
1797 dedupe: { coalesceKey: "{{body}}", coalesceWindowSeconds: 10 },
1798 },
1799 }),
1800 );
1801 yield* service.triggerWebhook(requestFor(task));
1802 assert.equal(yield* Queue.size(launches), 0);
1803 return task;
1804 }),
1805 { sql, workers: new Map() },
1806 );
1807 // The first service scope is closed before the deadline advances.
1808 yield* TestClock.adjust("30 seconds");
1809 const restartedWorkers = new Map<string, Effect.Effect<void>>();
1810 yield* withService(
1811 ({ service, launches }) =>
1812 Effect.gen(function* () {
1813 yield* drainPendingDispatch(restartedWorkers);
1814 yield* Queue.take(launches);
1815 for (let i = 0; i < 3; i++) yield* restartedWorkers.get("webhook-coalescing")!;
1816 assert.equal(yield* Queue.size(launches), 0);
1817 assert.equal((yield* service.list()).tasks[0]!.runCount, 1);
1818 assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0);
1819 }),
1820 { sql, workers: restartedWorkers },
1821 );
1822 const secondRestartWorkers = new Map<string, Effect.Effect<void>>();
1823 yield* withService(
1824 ({ service, launches }) =>
1825 Effect.gen(function* () {
1826 yield* secondRestartWorkers.get("webhook-coalescing")!;
1827 assert.equal(yield* Queue.size(launches), 0);
1828 assert.equal(
1829 (yield* service.list()).tasks.find((entry) => entry.id === task.id)!.runCount,
1830 1,
1831 );
1832 }),
1833 { sql, workers: secondRestartWorkers },
1834 );
1835 }).pipe(Effect.provide(SqlitePersistence.layerMemory)),
1836 );
1837
1838 it.effect("removes a window with a render defect and dispatches the task's next window", () => {
1839 const workers = new Map<string, Effect.Effect<void>>();
1840 return withService(
1841 ({ service, sql, launches }) =>
1842 Effect.gen(function* () {
1843 const { task } = yield* service.upsert(
1844 yield* webhookTaskInput({
1845 prompt: "{{body.deep}}",
1846 schedule: {
1847 type: "webhook",
1848 dedupe: {
1849 replayKey: "{{headers.delivery-id}}",
1850 coalesceKey: "{{headers.group}}",
1851 coalesceWindowSeconds: 10,
1852 },
1853 },
1854 }),
1855 );
1856 const request = (id: string, bodyText: string) =>
1857 requestFor(task, {
1858 bodyText,
1859 body: new TextEncoder().encode(bodyText),
1860 headers: { "delivery-id": id, group: id },
1861 relayDeliveryId: id,
1862 });
1863 const bad = request(
1864 "bad-render",
1865 '{"deep":' + "[".repeat(20000) + "0" + "]".repeat(20000) + "}",
1866 );
1867 assert.throws(() => renderWebhookPrompt(task.prompt, bad, "frame"), RangeError);
1868 const accepted = yield* service.triggerWebhook(bad);
1869 assert.equal(accepted._tag, "accepted");
1870 yield* service.triggerWebhook(request("good-render", '{"deep":"next window"}'));
1871 yield* TestClock.adjust("10 seconds");
1872 yield* drainPendingDispatch(workers);
1873 assert.include((yield* Queue.take(launches)).initialMessage!.text, "next window");
1874 assert.equal(yield* Queue.size(launches), 0);
1875 assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0);
1876 assert.equal(
1877 (yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys WHERE delivery_id = 'delivery:relay:bad-render'`)
1878 .length,
1879 0,
1880 );
1881 assert.equal(
1882 (yield* sql`SELECT 1 FROM scheduled_task_webhook_relay_deliveries WHERE relay_delivery_id = 'bad-render'`)
1883 .length,
1884 0,
1885 );
1886 const failed = (yield* service.getWebhookDelivery({
1887 id: task.id,
1888 deliveryId: ScheduledTaskWebhookDeliveryId.make("delivery:relay:bad-render"),
1889 acceptAnyOutcome: true,
1890 })).delivery;
1891 assert.equal(failed.outcome, "dispatch_failed");
1892 assert.equal(failed.error, "The pending webhook payload could not be prepared.");
1893 yield* workers.get("webhook-coalescing")!;
1894 assert.equal((yield* service.list()).tasks[0]!.runCount, 1);
1895 }),
1896 { workers },
1897 );
1898 });
1899
1900 it.effect.each([false, true])(
1901 "counts an interrupted immediate run beside an unrelated running window (corrupt task: %s)",
1902 (corrupt) => {
1903 const workers = new Map<string, Effect.Effect<void>>();
1904 const completedLaunches = new Set<string>();
1905 return Effect.gen(function* () {
1906 const gate = yield* Deferred.make<void>();
1907 yield* withService(
1908 ({ service, sql, launches }) =>
1909 Effect.gen(function* () {
1910 const { task } = yield* service.upsert(
1911 yield* webhookTaskInput({
1912 prompt: "{{body.value}}",
1913 schedule: {
1914 type: "webhook",
1915 dedupe: { coalesceKey: "{{headers.group}}", coalesceWindowSeconds: 10 },
1916 },
1917 }),
1918 );
1919 const window = yield* service.triggerWebhook(
1920 requestFor(task, { headers: { group: "old" }, bodyText: '{"value":"old"}' }),
1921 );
1922 yield* sql`CREATE TRIGGER fail_old_completion BEFORE UPDATE ON scheduled_tasks
1923 WHEN NEW.last_run_status = 'succeeded' BEGIN SELECT RAISE(FAIL, 'completion unavailable'); END`;
1924 yield* TestClock.adjust("10 seconds");
1925 yield* drainPendingDispatch(workers);
1926 yield* Queue.take(launches);
1927 yield* sql`DROP TRIGGER fail_old_completion`;
1928 assert.equal(
1929 (yield* sql`SELECT 1 FROM scheduled_task_webhook_pending WHERE state = 'running'`)
1930 .length,
1931 1,
1932 );
1933 const immediate = yield* service.triggerWebhook(
1934 requestFor(task, { bodyText: '{"value":"immediate"}' }),
1935 );
1936 yield* Queue.take(launches);
1937 const [before] = yield* sql<{
1938 last_run_webhook_delivery_id: string;
1939 model_selection_json: string;
1940 run_count: number;
1941 }>`SELECT last_run_webhook_delivery_id, model_selection_json, run_count FROM scheduled_tasks WHERE task_id = ${task.id}`;
1942 assert.equal(
1943 before!.last_run_webhook_delivery_id,
1944 immediate._tag === "accepted" ? immediate.deliveryId : "",
1945 );
1946 assert.notEqual(
1947 before!.last_run_webhook_delivery_id,
1948 window._tag === "accepted" ? window.deliveryId : "",
1949 );
1950 assert.equal(before!.run_count, 0);
1951 if (corrupt)
1952 yield* sql`UPDATE scheduled_tasks SET model_selection_json = 'broken' WHERE task_id = ${task.id}`;
1953 const restartedWorkers = new Map<string, Effect.Effect<void>>();
1954 yield* withService(
1955 ({ service: restarted, launches: recoveredLaunches }) =>
1956 Effect.gen(function* () {
1957 const [recovered] = yield* sql<{
1958 run_count: number;
1959 last_run_status: string;
1960 }>`SELECT run_count, last_run_status FROM scheduled_tasks WHERE task_id = ${task.id}`;
1961 assert.equal(recovered!.run_count, 1);
1962 assert.equal(recovered!.last_run_status, "failed");
1963 if (corrupt)
1964 yield* sql`UPDATE scheduled_tasks SET model_selection_json = ${before!.model_selection_json} WHERE task_id = ${task.id}`;
1965 yield* drainPendingDispatch(restartedWorkers);
1966 assert.equal(yield* Queue.size(recoveredLaunches), 0);
1967 assert.equal((yield* restarted.list()).tasks[0]!.runCount, 2);
1968 assert.equal(
1969 (yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length,
1970 0,
1971 );
1972 yield* restartedWorkers.get("webhook-coalescing")!;
1973 assert.equal((yield* restarted.list()).tasks[0]!.runCount, 2);
1974 }),
1975 { sql, workers: restartedWorkers, completedLaunches },
1976 );
1977 }),
1978 {
1979 workers,
1980 completedLaunches,
1981 gateForTask: (input) =>
1982 input.initialMessage?.text.includes(">immediate<") ? gate : undefined,
1983 },
1984 );
1985 });
1986 },
1987 );
1988
1989 it.effect("checks filters after the signature and max age, and runs only matching deliveries", () =>
1990 withService(({ service, launches }) =>
1991 Effect.gen(function* () {
1992 const { task } = yield* service.upsert(
1993 yield* webhookTaskInput({
1994 schedule: {
1995 type: "webhook",
1996 signature: {
1997 header: "x-hub-signature-256",
1998 encoding: "hex",
1999 prefix: "sha256=",
2000 secret: "s3cret",
2001 },
2002 maxDeliveryAgeMinutes: 30,
2003 filters: [
2004 { path: "headers.x-github-event", op: "equals", value: "pull_request" },
2005 { path: "body.action", op: "in", value: ["opened", "reopened"] },
2006 ],
2007 },
2008 }),
2009 );
2010 const delivery = (
2011 event: string,
2012 action: string,
2013 options: { readonly signed?: boolean; readonly receivedAt?: string } = {},
2014 ) => {
2015 const body = new TextEncoder().encode(
2016 JSON.stringify({ action, pull_request: { url: "https://github.com/org/repo/pull/45" } }),
2017 );
2018 const signature = `sha256=${NodeCrypto.createHmac("sha256", "s3cret").update(body).digest("hex")}`;
2019 return requestFor(task, {
2020 headers: {
2021 "content-type": "application/json",
2022 "x-github-event": event,
2023 ...(options.signed === false ? {} : { "x-hub-signature-256": signature }),
2024 },
2025 body,
2026 bodyText: new TextDecoder().decode(body),
2027 ...(options.receivedAt === undefined
2028 ? {}
2029 : { relayDeliveryId: "old", receivedAt: options.receivedAt }),
2030 });
2031 };
2032 const now = yield* DateTime.now;
2033 const old = DateTime.formatIso(DateTime.subtract(now, { minutes: 31 }));
2034
2035 // A request failing a filter is still refused first for its signature or age.
2036 const unsigned = yield* service.triggerWebhook(delivery("push", "opened", { signed: false }));
2037 assert.equal(unsigned._tag, "rejected_signature");
2038 const tooOld = yield* service.triggerWebhook(delivery("push", "opened", { receivedAt: old }));
2039 assert.equal(tooOld._tag, "expired");
2040
2041 const wrongEvent = yield* service.triggerWebhook(delivery("push", "opened"));
2042 const wrongAction = yield* service.triggerWebhook(delivery("pull_request", "closed"));
2043 assert.deepInclude(wrongEvent, { _tag: "accepted", outcome: "filtered" });
2044 assert.deepInclude(wrongAction, { _tag: "accepted", outcome: "filtered" });
2045 assert.equal(yield* Queue.size(launches), 0);
2046 const [listed] = (yield* service.list()).tasks;
2047 assert.equal(listed?.runCount, 0);
2048 assert.equal(listed?.lastRunStatus, "never");
2049
2050 const matched = yield* service.triggerWebhook(delivery("pull_request", "opened"));
2051 assert.deepInclude(matched, { _tag: "accepted", outcome: "accepted" });
2052 assert.include((yield* Queue.take(launches)).initialMessage?.text ?? "", "pull/45");
2053
2054 const deliveries = (yield* service.listWebhookDeliveries({
2055 id: task.id,
2056 acceptAnyOutcome: true,
2057 })).deliveries;
2058 const filtered = deliveries
2059 .filter((entry) => entry.outcome === "filtered")
2060 .map((entry) => [entry.signatureVerified, entry.error]);
2061 assert.sameDeepMembers(filtered, [
2062 [true, "Filter 1 (headers.x-github-event) did not match."],
2063 [true, "Filter 2 (body.action) did not match."],
2064 ]);
2065 }),
2066 ),
2067 );
2068
2069 const pullRequestFilterInput = () =>
2070 webhookTaskInput({
2071 schedule: {
2072 type: "webhook",
2073 filters: [{ path: "headers.x-github-event", op: "equals", value: "pull_request" }],
2074 },
2075 });
2076 const githubEvent = (event: string) => ({
2077 headers: { "content-type": "application/json", "x-github-event": event },
2078 });
2079
2080 it.effect("a filtered relay delivery releases its claim, so a redelivery is evaluated again", () =>
2081 withService(({ service, launches }) =>
2082 Effect.gen(function* () {
2083 const { task } = yield* service.upsert(yield* pullRequestFilterInput());
2084 const held = { relayDeliveryId: "relay-filtered" };
2085 const first = yield* service.triggerWebhook(
2086 requestFor(task, { ...held, ...githubEvent("push") }),
2087 );
2088 const again = yield* service.triggerWebhook(
2089 requestFor(task, { ...held, ...githubEvent("push") }),
2090 );
2091 assert.deepInclude(first, { _tag: "accepted", outcome: "filtered" });
2092 assert.deepInclude(again, { _tag: "accepted", outcome: "filtered" });
2093 const ran = yield* service.triggerWebhook(
2094 requestFor(task, { ...held, ...githubEvent("pull_request") }),
2095 );
2096 assert.deepInclude(ran, { _tag: "accepted", outcome: "accepted" });
2097 yield* Queue.take(launches);
2098 const replayed = yield* service.triggerWebhook(
2099 requestFor(task, { ...held, ...githubEvent("pull_request") }),
2100 );
2101 assert.deepInclude(replayed, { _tag: "accepted", outcome: "duplicate" });
2102 }),
2103 ),
2104 );
2105
2106 it.effect("leaves filtered rows out of the list for clients that only decode older outcomes", () =>
2107 withService(({ service, launches }) =>
2108 Effect.gen(function* () {
2109 const { task } = yield* service.upsert(yield* pullRequestFilterInput());
2110 yield* service.triggerWebhook(requestFor(task, githubEvent("push")));
2111 yield* service.triggerWebhook(requestFor(task, githubEvent("pull_request")));
2112 yield* Queue.take(launches);
2113 const legacy = (yield* service.listWebhookDeliveries({ id: task.id })).deliveries;
2114 assert.deepEqual(
2115 legacy.map((delivery) => delivery.outcome),
2116 ["accepted"],
2117 );
2118 const current = (yield* service.listWebhookDeliveries({
2119 id: task.id,
2120 acceptAnyOutcome: true,
2121 })).deliveries;
2122 assert.sameMembers(
2123 current.map((delivery) => delivery.outcome),
2124 ["accepted", "filtered"],
2125 );
2126 }),
2127 ),
2128 );
2129
2130 it.effect("filters run before replay and coalescing without claiming ignored deliveries", () => {
2131 const workers = new Map<string, Effect.Effect<void>>();
2132 return withService(
2133 ({ service, launches, sql }) =>
2134 Effect.gen(function* () {
2135 const { task } = yield* service.upsert(
2136 yield* webhookTaskInput({
2137 prompt: "{{body.value}}",
2138 schedule: {
2139 type: "webhook",
2140 filters: [{ path: "body.run", op: "equals", value: "true" }],
2141 dedupe: {
2142 replayKey: "{{headers.delivery-id}}",
2143 coalesceKey: "{{body.group}}",
2144 coalesceWindowSeconds: 10,
2145 },
2146 },
2147 }),
2148 );
2149 const request = (id: string, run: boolean, value: string) => {
2150 const bodyText = JSON.stringify({ run, group: "B", value });
2151 return requestFor(task, {
2152 headers: { "delivery-id": id },
2153 bodyText,
2154 body: new TextEncoder().encode(bodyText),
2155 relayDeliveryId: id,
2156 });
2157 };
2158 const ignored = yield* service.triggerWebhook(request("A", false, "ignored"));
2159 assert.deepInclude(ignored, { _tag: "accepted", outcome: "filtered" });
2160 assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_replay_keys`).length, 0);
2161 assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_pending`).length, 0);
2162 assert.equal((yield* sql`SELECT 1 FROM scheduled_task_webhook_relay_deliveries`).length, 0);
2163 const filteredId = ignored._tag === "accepted" ? ignored.deliveryId : assert.fail();
2164 const oldClient = yield* service
2165 .getWebhookDelivery({ id: task.id, deliveryId: filteredId })
2166 .pipe(Effect.result);
2167 assert.equal(oldClient._tag, "Failure");
2168 const currentClient = yield* service.getWebhookDelivery({
2169 id: task.id,
2170 deliveryId: filteredId,
2171 acceptAnyOutcome: true,
2172 });
2173 assert.equal(currentClient.delivery.outcome, "filtered");
2174 assert.deepInclude(yield* service.triggerWebhook(request("A", true, "first")), {
2175 _tag: "accepted",
2176 outcome: "accepted",
2177 });
2178 assert.deepInclude(
2179 yield* service.triggerWebhook({
2180 ...request("A", false, "ignored replay"),
2181 relayDeliveryId: "ignored-replay",
2182 }),
2183 { _tag: "accepted", outcome: "filtered" },
2184 );
2185 assert.deepInclude(yield* service.triggerWebhook(request("B", false, "ignored update")), {
2186 _tag: "accepted",
2187 outcome: "filtered",
2188 });
2189 assert.deepInclude(yield* service.triggerWebhook(request("B", true, "latest")), {
2190 _tag: "accepted",
2191 outcome: "coalesced",
2192 });
2193 yield* TestClock.adjust("10 seconds");
2194 yield* drainPendingDispatch(workers);
2195 assert.include((yield* Queue.take(launches)).initialMessage!.text, "latest");
2196 assert.equal(yield* Queue.size(launches), 0);
2197 }),
2198 { workers },
2199 );
2200 });