The best way to control Claude Code, Codex and more from any computer or phone

main
ts 806 lines 29.7 KB
Raw
1 import {
2 CheckpointId,
3 CheckpointScopeId,
4 CommandId,
5 MessageId,
6 ModelSelection,
7 NodeId,
8 NonNegativeInt,
9 ProviderSessionId,
10 RunAttemptId,
11 ProviderApprovalDecision,
12 ProviderUserInputAnswers,
13 ProviderThreadId,
14 ProviderTurnId,
15 RunId,
16 RuntimeRequestId,
17 ThreadId,
18 } from "@t3tools/contracts";
19 import * as Context from "effect/Context";
20 import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient";
21 import * as DateTime from "effect/DateTime";
22 import * as Deferred from "effect/Deferred";
23 import * as Duration from "effect/Duration";
24 import * as Effect from "effect/Effect";
25 import * as Layer from "effect/Layer";
26 import * as Option from "effect/Option";
27 import * as Queue from "effect/Queue";
28 import * as Result from "effect/Result";
29 import * as Schedule from "effect/Schedule";
30 import * as Schema from "effect/Schema";
31 import * as SqlClient from "effect/sql/SqlClient";
32
33 import { forkParked } from "../serverActivation.ts";
34
35 export const OrchestrationEffectRequestV2 = Schema.Union([
36 Schema.Struct({
37 type: Schema.Literal("provider-runtime.continue"),
38 sourceRunId: RunId,
39 processLossCount: Schema.optional(Schema.Int.check(Schema.isGreaterThanOrEqualTo(0))),
40 }),
41 Schema.Struct({
42 type: Schema.Literal("provider-session.detach"),
43 providerSessionId: ProviderSessionId,
44 detail: Schema.optional(Schema.String),
45 revokeMcpCredential: Schema.optional(Schema.Boolean),
46 }),
47 Schema.Struct({
48 type: Schema.Literal("provider-session.apply-model-selection"),
49 providerSessionId: ProviderSessionId,
50 providerThreadId: ProviderThreadId,
51 modelSelection: ModelSelection,
52 }),
53 Schema.Struct({
54 type: Schema.Literal("provider-session.apply-thread-title"),
55 title: Schema.String,
56 }),
57 Schema.Struct({
58 type: Schema.Literal("provider-turn.start"),
59 runId: RunId,
60 }),
61 Schema.Struct({
62 type: Schema.Literal("provider-turn.interrupt"),
63 providerSessionId: ProviderSessionId,
64 providerThreadId: ProviderThreadId,
65 providerTurnId: ProviderTurnId,
66 subagent: Schema.optional(Schema.Struct({ id: NodeId, nativeTaskId: Schema.String })),
67 }),
68 Schema.Struct({
69 type: Schema.Literal("provider-turn.steer"),
70 providerSessionId: ProviderSessionId,
71 providerThreadId: ProviderThreadId,
72 providerTurnId: ProviderTurnId,
73 messageId: MessageId,
74 modelSelection: Schema.optional(ModelSelection),
75 steerFollowUpAfterRunOrdinal: Schema.optional(NonNegativeInt),
76 }),
77 Schema.Struct({
78 type: Schema.Literal("provider-turn.restart"),
79 providerSessionId: ProviderSessionId,
80 providerThreadId: ProviderThreadId,
81 providerTurnId: ProviderTurnId,
82 interruptedAttemptId: RunAttemptId,
83 runId: RunId,
84 sessionTransition: Schema.optional(
85 Schema.Union([
86 Schema.Struct({
87 type: Schema.Literal("replace"),
88 replacementProviderSessionId: ProviderSessionId,
89 }),
90 Schema.Struct({ type: Schema.Literal("detach") }),
91 ]),
92 ),
93 }),
94 Schema.Struct({
95 type: Schema.Literal("runtime-request.respond"),
96 providerSessionId: ProviderSessionId,
97 requestId: RuntimeRequestId,
98 decision: Schema.optional(ProviderApprovalDecision),
99 feedback: Schema.optional(Schema.String),
100 answers: Schema.optional(ProviderUserInputAnswers),
101 }),
102 Schema.Struct({
103 type: Schema.Literal("provider-thread.rollback"),
104 restoreFiles: Schema.optional(Schema.Boolean),
105 providerThreadId: ProviderThreadId,
106 checkpointId: CheckpointId,
107 scopeId: CheckpointScopeId,
108 }),
109 Schema.Struct({
110 type: Schema.Literal("checkpoint.capture"),
111 runId: RunId,
112 scopeId: CheckpointScopeId,
113 }),
114 Schema.Struct({
115 type: Schema.Literal("scheduled-tasks.cleanup"),
116 }),
117 Schema.Struct({
118 type: Schema.Literal("terminal.cleanup"),
119 }),
120 Schema.Struct({
121 type: Schema.Literal("preview.cleanup"),
122 }),
123 Schema.Struct({
124 type: Schema.Literal("attachment.cleanup"),
125 attachmentIds: Schema.Array(Schema.String),
126 skipReferenced: Schema.optional(Schema.Boolean),
127 }),
128 Schema.Struct({
129 type: Schema.Literal("thread-title.generate"),
130 kind: Schema.Union([
131 Schema.Struct({ type: Schema.Literal("initial"), messageId: MessageId }),
132 Schema.Struct({ type: Schema.Literal("regenerate") }),
133 ]),
134 }),
135 Schema.Struct({
136 type: Schema.Literal("delegated-tasks.stop"),
137 reason: Schema.optional(Schema.String),
138 }),
139 ]);
140 export type OrchestrationEffectRequestV2 = typeof OrchestrationEffectRequestV2.Type;
141
142 export const REPLAY_SAFE_EFFECT_TYPES_AFTER_PROCESS_LOSS = [
143 "provider-runtime.continue",
144 "provider-session.detach",
145 "provider-thread.rollback",
146 "checkpoint.capture",
147 "scheduled-tasks.cleanup",
148 "terminal.cleanup",
149 "preview.cleanup",
150 "attachment.cleanup",
151 "thread-title.generate",
152 "delegated-tasks.stop",
153 ] as const satisfies ReadonlyArray<OrchestrationEffectRequestV2["type"]>;
154
155 export const PROCESS_BOUND_EFFECT_TYPES = [
156 "provider-session.apply-model-selection",
157 "provider-session.apply-thread-title",
158 "provider-turn.start",
159 "provider-turn.interrupt",
160 "provider-turn.steer",
161 "provider-turn.restart",
162 "runtime-request.respond",
163 ] as const satisfies ReadonlyArray<OrchestrationEffectRequestV2["type"]>;
164
165 export const OrchestrationEffectStatusV2 = Schema.Literals([
166 "pending",
167 "running",
168 "succeeded",
169 "failed",
170 "cancelled",
171 ]);
172 export type OrchestrationEffectStatusV2 = typeof OrchestrationEffectStatusV2.Type;
173
174 /** Receipts deduplicate retries, and restart re-enqueues need far less than a week. Retain failed rows to keep deleted-thread worktrees. */
175 export const SETTLED_EFFECT_RETENTION = Duration.days(7);
176 const PRUNE_BATCH_SIZE = 500;
177
178 export interface OrchestrationEffectV2 {
179 readonly id: string;
180 readonly commandId: CommandId;
181 readonly threadId: ThreadId;
182 readonly request: OrchestrationEffectRequestV2;
183 readonly status: OrchestrationEffectStatusV2;
184 readonly attemptCount: number;
185 readonly availableAt: string;
186 readonly leaseOwner: string | null;
187 readonly createdAt: string;
188 readonly updatedAt: string;
189 readonly completedAt: string | null;
190 readonly lastError: string | null;
191 }
192
193 export interface PendingOrchestrationEffectV2 {
194 readonly id: string;
195 readonly commandId: CommandId;
196 readonly threadId: ThreadId;
197 readonly request: OrchestrationEffectRequestV2;
198 readonly availableAt?: DateTime.Utc;
199 }
200
201 export class EffectOutboxError extends Schema.TaggedError<EffectOutboxError>()(
202 "EffectOutboxError",
203 {
204 operation: Schema.String,
205 effectId: Schema.optional(Schema.String),
206 cause: Schema.optional(Schema.Defect()),
207 },
208 ) {
209 override get message(): string {
210 return `Orchestration effect outbox ${this.operation} failed${this.effectId === undefined ? "" : ` for ${this.effectId}`}.`;
211 }
212 }
213
214 const isEffectOutboxError = Schema.is(EffectOutboxError);
215
216 export interface EffectOutboxV2Shape {
217 readonly awaitAvailable: Effect.Effect<void>;
218 readonly notifyAvailable: (count?: number) => Effect.Effect<void>;
219 /** Persist rows only. Notify workers after the surrounding transaction commits. */
220 readonly enqueue: (
221 effects: ReadonlyArray<PendingOrchestrationEffectV2>,
222 ) => Effect.Effect<void, EffectOutboxError>;
223 readonly get: (
224 effectId: string,
225 ) => Effect.Effect<Option.Option<OrchestrationEffectV2>, EffectOutboxError>;
226 readonly listByCommandId: (
227 commandId: CommandId,
228 ) => Effect.Effect<ReadonlyArray<OrchestrationEffectV2>, EffectOutboxError>;
229 /** A message's first steer effect: it records which runs were queued before the message. */
230 readonly firstSteer: (
231 threadId: ThreadId,
232 messageId: MessageId,
233 ) => Effect.Effect<Option.Option<OrchestrationEffectV2>, EffectOutboxError>;
234 readonly cancelUnsettled: (input: {
235 readonly threadId: ThreadId;
236 readonly effectTypes: ReadonlyArray<OrchestrationEffectRequestV2["type"]>;
237 readonly reason: string;
238 }) => Effect.Effect<ReadonlyArray<string>, EffectOutboxError>;
239 readonly signalCancellations: (effectIds: ReadonlyArray<string>) => Effect.Effect<void>;
240 readonly awaitCancellation: (effectId: string) => Effect.Effect<void>;
241 readonly clearCancellation: (effectId: string) => Effect.Effect<void>;
242 /** Resolves once the effect is neither pending nor running, or right away when it does not exist. */
243 readonly awaitSettled: (effectId: string) => Effect.Effect<void, EffectOutboxError>;
244 readonly reconcileAfterProcessLoss: Effect.Effect<
245 { readonly requeued: number; readonly cancelled: number },
246 EffectOutboxError
247 >;
248 readonly claimNext: (input: {
249 readonly workerId: string;
250 readonly excludeRestartContinuations?: boolean;
251 readonly maxRunningTitleGenerations?: number;
252 }) => Effect.Effect<Option.Option<OrchestrationEffectV2>, EffectOutboxError>;
253 readonly nextClaimableAt: (input?: {
254 readonly maxRunningTitleGenerations?: number;
255 }) => Effect.Effect<Option.Option<DateTime.Utc>, EffectOutboxError>;
256 readonly succeed: (input: {
257 readonly effectId: string;
258 readonly workerId: string;
259 }) => Effect.Effect<boolean, EffectOutboxError>;
260 readonly retry: (input: {
261 readonly effectId: string;
262 readonly workerId: string;
263 readonly error: string;
264 readonly delayMs: number;
265 }) => Effect.Effect<boolean, EffectOutboxError>;
266 readonly fail: (input: {
267 readonly effectId: string;
268 readonly workerId: string;
269 readonly error: string;
270 }) => Effect.Effect<boolean, EffectOutboxError>;
271 readonly pruneSettled: Effect.Effect<number, EffectOutboxError>;
272 }
273
274 export class EffectOutboxV2 extends Context.Service<EffectOutboxV2, EffectOutboxV2Shape>()(
275 "t3/orchestration-v2/EffectOutbox/EffectOutboxV2",
276 ) {}
277
278 type EffectRow = {
279 readonly effect_id: string;
280 readonly command_id: string;
281 readonly thread_id: string;
282 readonly effect_type: string;
283 readonly payload_json: string;
284 readonly status: string;
285 readonly attempt_count: number;
286 readonly available_at: string;
287 readonly lease_owner: string | null;
288 readonly created_at: string;
289 readonly updated_at: string;
290 readonly completed_at: string | null;
291 readonly last_error: string | null;
292 };
293
294 const encodeRequest = Schema.encodeSync(Schema.fromJsonString(OrchestrationEffectRequestV2));
295 const decodeRequest = Schema.decodeUnknownEffect(
296 Schema.fromJsonString(OrchestrationEffectRequestV2),
297 );
298
299 const rowToEffect = (row: EffectRow) =>
300 decodeRequest(row.payload_json).pipe(
301 Effect.map((request): OrchestrationEffectV2 => ({
302 id: row.effect_id,
303 commandId: CommandId.make(row.command_id),
304 threadId: ThreadId.make(row.thread_id),
305 request,
306 status: row.status as OrchestrationEffectStatusV2,
307 attemptCount: row.attempt_count,
308 availableAt: row.available_at,
309 leaseOwner: row.lease_owner,
310 createdAt: row.created_at,
311 updatedAt: row.updated_at,
312 completedAt: row.completed_at,
313 lastError: row.last_error,
314 })),
315 );
316
317 export const layer: Layer.Layer<EffectOutboxV2, never, SqlClient.SqlClient> = Layer.effect(
318 EffectOutboxV2,
319 Effect.gen(function* () {
320 const sql = yield* SqlClient.SqlClient;
321 const available = yield* Queue.dropping<void>(64);
322 const cancellationSignals = new Map<string, Deferred.Deferred<void>>();
323 const settledSignals = new Map<string, Deferred.Deferred<void>>();
324 const signalSettled = (effectIds: ReadonlyArray<string>) =>
325 Effect.forEach(
326 effectIds,
327 (effectId) => {
328 const signal = settledSignals.get(effectId);
329 if (signal === undefined) return Effect.void;
330 settledSignals.delete(effectId);
331 return Deferred.succeed(signal, undefined);
332 },
333 { discard: true },
334 );
335 const notifyAvailable = (count = 1) =>
336 Queue.offerAll(
337 available,
338 Array.from({ length: Math.min(64, Math.max(0, Math.floor(count))) }, () => undefined),
339 ).pipe(Effect.asVoid);
340 // Retry backoff must block later effects so turns cannot race a pending rollback.
341 const claimableCandidatePredicate = (
342 availableBefore?: string,
343 excludeRestartContinuations = false,
344 maxRunningTitleGenerations?: number,
345 ) =>
346 sql`
347 ${
348 availableBefore === undefined
349 ? sql`1 = 1`
350 : sql`candidate.available_at <= ${availableBefore}`
351 }
352 AND candidate.status = 'pending'
353 AND NOT EXISTS (
354 SELECT 1
355 FROM orchestration_v2_effect_outbox AS active
356 WHERE active.thread_id = candidate.thread_id
357 AND (
358 active.status = 'running'
359 OR (
360 active.status = 'pending'
361 AND active.rowid < candidate.rowid
362 AND ${
363 excludeRestartContinuations
364 ? sql`active.effect_type != 'provider-runtime.continue'`
365 : sql`1 = 1`
366 }
367 )
368 )
369 AND (
370 (
371 candidate.effect_type = 'thread-title.generate'
372 AND active.effect_type = 'thread-title.generate'
373 )
374 OR
375 (
376 candidate.effect_type != 'thread-title.generate'
377 AND active.effect_type != 'thread-title.generate'
378 )
379 )
380 )
381 AND ${
382 maxRunningTitleGenerations === undefined
383 ? sql`1 = 1`
384 : sql`(
385 candidate.effect_type != 'thread-title.generate'
386 OR (
387 SELECT COUNT(*)
388 FROM orchestration_v2_effect_outbox AS running_title
389 WHERE running_title.status = 'running'
390 AND running_title.effect_type = 'thread-title.generate'
391 ) < ${maxRunningTitleGenerations}
392 )`
393 }
394 `;
395
396 const cancellationSignal = (effectId: string) => {
397 const existing = cancellationSignals.get(effectId);
398 if (existing !== undefined) return existing;
399 const created = Deferred.makeUnsafe<void>();
400 cancellationSignals.set(effectId, created);
401 return created;
402 };
403
404 const decodeRows = (operation: string, rows: ReadonlyArray<EffectRow>) =>
405 Effect.forEach(rows, rowToEffect).pipe(
406 Effect.mapError((cause) => new EffectOutboxError({ operation, cause })),
407 );
408
409 const service: EffectOutboxV2Shape = {
410 enqueue: Effect.fn("EffectOutbox.enqueue")(
411 function* (effects) {
412 const now = yield* DateTime.now;
413 const nowIso = DateTime.formatIso(now);
414 yield* Effect.forEach(
415 effects,
416 (effect) => sql`
417 INSERT INTO orchestration_v2_effect_outbox (
418 effect_id,
419 command_id,
420 thread_id,
421 effect_type,
422 payload_json,
423 status,
424 attempt_count,
425 available_at,
426 created_at,
427 updated_at
428 )
429 VALUES (
430 ${effect.id},
431 ${effect.commandId},
432 ${effect.threadId},
433 ${effect.request.type},
434 ${encodeRequest(effect.request)},
435 'pending',
436 0,
437 ${DateTime.formatIso(effect.availableAt ?? now)},
438 ${nowIso},
439 ${nowIso}
440 )
441 ON CONFLICT(effect_id) DO NOTHING
442 `,
443 { concurrency: 1, discard: true },
444 );
445 // EventSink owns the corresponding post-commit notification.
446 },
447 Effect.mapError((cause) => new EffectOutboxError({ operation: "enqueue", cause })),
448 ),
449 get: (effectId) =>
450 sql<EffectRow>`
451 SELECT *
452 FROM orchestration_v2_effect_outbox
453 WHERE effect_id = ${effectId}
454 LIMIT 1
455 `.pipe(
456 NodeSqliteClient.readOnly,
457 Effect.flatMap((rows) => {
458 const row = rows[0];
459 return row === undefined ? Effect.succeedNone : rowToEffect(row).pipe(Effect.asSome);
460 }),
461 Effect.mapError((cause) => new EffectOutboxError({ operation: "get", effectId, cause })),
462 ),
463 awaitAvailable: Queue.take(available),
464 notifyAvailable,
465 listByCommandId: (commandId) =>
466 sql<EffectRow>`
467 SELECT *
468 FROM orchestration_v2_effect_outbox
469 WHERE command_id = ${commandId}
470 ORDER BY created_at ASC, effect_id ASC
471 `.pipe(
472 NodeSqliteClient.readOnly,
473 Effect.flatMap((rows) => decodeRows("list", rows)),
474 Effect.mapError((cause) =>
475 isEffectOutboxError(cause)
476 ? cause
477 : new EffectOutboxError({ operation: "list", cause }),
478 ),
479 ),
480 firstSteer: (threadId, messageId) =>
481 sql<EffectRow>`
482 SELECT *
483 FROM orchestration_v2_effect_outbox
484 WHERE thread_id = ${threadId}
485 AND effect_type = 'provider-turn.steer'
486 AND json_extract(payload_json, '$.messageId') = ${messageId}
487 ORDER BY rowid ASC
488 LIMIT 1
489 `.pipe(
490 NodeSqliteClient.readOnly,
491 Effect.flatMap((rows) => decodeRows("first-steer", rows)),
492 Effect.map((effects) => Option.fromNullishOr(effects[0])),
493 Effect.mapError((cause) =>
494 isEffectOutboxError(cause)
495 ? cause
496 : new EffectOutboxError({ operation: "first-steer", cause }),
497 ),
498 ),
499 cancelUnsettled: Effect.fn("EffectOutbox.cancelUnsettled")(
500 function* ({ threadId, effectTypes, reason }) {
501 if (effectTypes.length === 0) return [];
502 const now = DateTime.formatIso(yield* DateTime.now);
503 const rows = yield* sql<{ readonly effect_id: string }>`
504 UPDATE orchestration_v2_effect_outbox
505 SET
506 status = 'cancelled',
507 lease_owner = NULL,
508 completed_at = ${now},
509 updated_at = ${now},
510 last_error = ${reason}
511 WHERE thread_id = ${threadId}
512 AND status IN ('pending', 'running')
513 AND effect_type IN ${sql.in(effectTypes)}
514 RETURNING effect_id
515 `;
516 return rows.map(({ effect_id }) => effect_id);
517 },
518 Effect.mapError((cause) => new EffectOutboxError({ operation: "cancel-unsettled", cause })),
519 ),
520 signalCancellations: Effect.fn("EffectOutbox.signalCancellations")(function* (effectIds) {
521 yield* Effect.forEach(
522 effectIds,
523 (effectId) => {
524 const signal = cancellationSignals.get(effectId);
525 return signal === undefined ? Effect.void : Deferred.succeed(signal, undefined);
526 },
527 { discard: true },
528 );
529 yield* signalSettled(effectIds);
530 if (effectIds.length > 0) yield* notifyAvailable(effectIds.length);
531 }),
532 awaitCancellation: (effectId) => Deferred.await(cancellationSignal(effectId)),
533 clearCancellation: (effectId) =>
534 Effect.sync(() => {
535 cancellationSignals.delete(effectId);
536 }),
537 awaitSettled: Effect.fn("EffectOutbox.awaitSettled")(function* (effectId) {
538 // Registered before the read, so a settle right after it still resolves.
539 const signal = settledSignals.get(effectId) ?? Deferred.makeUnsafe<void>();
540 settledSignals.set(effectId, signal);
541 const effect = yield* service.get(effectId);
542 if (Option.exists(effect, ({ status }) => status === "pending" || status === "running")) {
543 return yield* Deferred.await(signal);
544 }
545 yield* signalSettled([effectId]);
546 }),
547 reconcileAfterProcessLoss: Effect.gen(function* () {
548 const now = DateTime.formatIso(yield* DateTime.now);
549 const cancelledRows = yield* sql<{ readonly effect_id: string }>`
550 UPDATE orchestration_v2_effect_outbox
551 SET
552 status = 'cancelled',
553 lease_owner = NULL,
554 completed_at = ${now},
555 updated_at = ${now},
556 last_error = 'Cancelled because the server process ended before the effect completed.'
557 WHERE status IN ('pending', 'running')
558 AND effect_type IN ${sql.in(PROCESS_BOUND_EFFECT_TYPES)}
559 RETURNING effect_id
560 `;
561 const requeuedRows = yield* sql<{ readonly effect_id: string }>`
562 UPDATE orchestration_v2_effect_outbox
563 SET
564 status = 'pending',
565 lease_owner = NULL,
566 available_at = ${now},
567 updated_at = ${now},
568 last_error = 'Requeued after the previous server process ended.'
569 WHERE status = 'running'
570 AND effect_type IN ${sql.in(REPLAY_SAFE_EFFECT_TYPES_AFTER_PROCESS_LOSS)}
571 RETURNING effect_id
572 `;
573 if (requeuedRows.length > 0) yield* notifyAvailable(requeuedRows.length);
574 return { requeued: requeuedRows.length, cancelled: cancelledRows.length };
575 }).pipe(
576 Effect.mapError(
577 (cause) => new EffectOutboxError({ operation: "reconcile-process-loss", cause }),
578 ),
579 ),
580 // The worker polls for the next effect even when there is no work to claim.
581 claimNext: Effect.fnUntraced(
582 function* ({ workerId, excludeRestartContinuations = false, maxRunningTitleGenerations }) {
583 const now = yield* DateTime.now;
584 const nowIso = DateTime.formatIso(now);
585 // Idle claims return empty; tracing their full SQL on every poll overwhelms useful traces.
586 const claimStatement = sql<EffectRow>`
587 UPDATE orchestration_v2_effect_outbox
588 SET
589 status = 'running',
590 attempt_count = attempt_count + 1,
591 lease_owner = ${workerId},
592 updated_at = ${nowIso},
593 last_error = NULL
594 WHERE effect_id = (
595 SELECT candidate.effect_id
596 FROM orchestration_v2_effect_outbox AS candidate
597 WHERE ${claimableCandidatePredicate(
598 nowIso,
599 excludeRestartContinuations,
600 maxRunningTitleGenerations,
601 )}
602 AND ${excludeRestartContinuations ? sql`candidate.effect_type != 'provider-runtime.continue'` : sql`1 = 1`}
603 ORDER BY candidate.available_at ASC, candidate.created_at ASC, candidate.effect_id ASC
604 LIMIT 1
605 )
606 RETURNING *
607 `;
608 while (true) {
609 const rows = yield* claimStatement.pipe(Effect.withTracerEnabled(false));
610 const row = rows[0];
611 if (row === undefined) return Option.none();
612 const decoded = yield* Effect.result(rowToEffect(row));
613 if (Result.isSuccess(decoded)) {
614 cancellationSignals.set(row.effect_id, Deferred.makeUnsafe<void>());
615 return Option.some(decoded.success);
616 }
617 // Left running, an unreadable row would block every later effect on its thread.
618 yield* Effect.logWarning("Failing an orchestration effect this build cannot decode", {
619 effectId: row.effect_id,
620 effectType: row.effect_type,
621 cause: decoded.failure,
622 });
623 yield* service.fail({
624 effectId: row.effect_id,
625 workerId,
626 error: `This server build cannot decode the ${row.effect_type} effect.`,
627 });
628 }
629 },
630 Effect.mapError((cause) =>
631 isEffectOutboxError(cause) ? cause : new EffectOutboxError({ operation: "claim", cause }),
632 ),
633 ),
634 nextClaimableAt: Effect.fnUntraced(
635 function* ({
636 maxRunningTitleGenerations,
637 }: { readonly maxRunningTitleGenerations?: number } = {}) {
638 const rows = yield* sql<{ readonly available_at: string | null }>`
639 SELECT MIN(candidate.available_at) AS available_at
640 FROM orchestration_v2_effect_outbox AS candidate
641 WHERE ${claimableCandidatePredicate(undefined, false, maxRunningTitleGenerations)}
642 `.pipe(NodeSqliteClient.readOnly, Effect.withTracerEnabled(false));
643 const availableAt = rows[0]?.available_at;
644 if (availableAt === undefined || availableAt === null) return Option.none();
645 const parsed = DateTime.make(availableAt);
646 if (Option.isNone(parsed)) {
647 return yield* new EffectOutboxError({
648 operation: "next-claimable",
649 cause: `Invalid available_at timestamp: ${availableAt}`,
650 });
651 }
652 return parsed;
653 },
654 Effect.mapError((cause) =>
655 isEffectOutboxError(cause)
656 ? cause
657 : new EffectOutboxError({ operation: "next-claimable", cause }),
658 ),
659 ),
660 succeed: Effect.fn("EffectOutbox.succeed")(
661 function* ({ effectId, workerId }) {
662 const now = DateTime.formatIso(yield* DateTime.now);
663 const rows = yield* sql<{ readonly effect_id: string }>`
664 UPDATE orchestration_v2_effect_outbox
665 SET
666 status = 'succeeded',
667 lease_owner = NULL,
668 completed_at = ${now},
669 updated_at = ${now},
670 last_error = NULL
671 WHERE effect_id = ${effectId}
672 AND status = 'running'
673 AND lease_owner = ${workerId}
674 RETURNING effect_id
675 `;
676 if (rows.length === 1) {
677 cancellationSignals.delete(effectId);
678 yield* signalSettled([effectId]);
679 yield* notifyAvailable();
680 }
681 return rows.length === 1;
682 },
683 (effect, { effectId }) =>
684 effect.pipe(
685 Effect.mapError(
686 (cause) => new EffectOutboxError({ operation: "succeed", effectId, cause }),
687 ),
688 ),
689 ),
690 retry: Effect.fn("EffectOutbox.retry")(
691 function* ({ effectId, workerId, error, delayMs }) {
692 const now = yield* DateTime.now;
693 const nowIso = DateTime.formatIso(now);
694 const availableAt = DateTime.formatIso(
695 DateTime.add(now, { milliseconds: Math.max(0, delayMs) }),
696 );
697 const rows = yield* sql<{ readonly effect_id: string }>`
698 UPDATE orchestration_v2_effect_outbox
699 SET
700 status = 'pending',
701 available_at = ${availableAt},
702 lease_owner = NULL,
703 updated_at = ${nowIso},
704 last_error = ${error}
705 WHERE effect_id = ${effectId}
706 AND status = 'running'
707 AND lease_owner = ${workerId}
708 RETURNING effect_id
709 `;
710 if (rows.length === 1) {
711 cancellationSignals.delete(effectId);
712 yield* notifyAvailable();
713 }
714 return rows.length === 1;
715 },
716 (effect, { effectId }) =>
717 effect.pipe(
718 Effect.mapError(
719 (cause) => new EffectOutboxError({ operation: "retry", effectId, cause }),
720 ),
721 ),
722 ),
723 fail: Effect.fn("EffectOutbox.fail")(
724 function* ({ effectId, workerId, error }) {
725 const now = DateTime.formatIso(yield* DateTime.now);
726 const rows = yield* sql<{ readonly effect_id: string }>`
727 UPDATE orchestration_v2_effect_outbox
728 SET
729 status = 'failed',
730 lease_owner = NULL,
731 completed_at = ${now},
732 updated_at = ${now},
733 last_error = ${error}
734 WHERE effect_id = ${effectId}
735 AND status = 'running'
736 AND lease_owner = ${workerId}
737 RETURNING effect_id
738 `;
739 if (rows.length === 1) {
740 cancellationSignals.delete(effectId);
741 yield* signalSettled([effectId]);
742 yield* notifyAvailable();
743 }
744 return rows.length === 1;
745 },
746 (effect, { effectId }) =>
747 effect.pipe(
748 Effect.mapError(
749 (cause) => new EffectOutboxError({ operation: "fail", effectId, cause }),
750 ),
751 ),
752 ),
753 pruneSettled: Effect.gen(function* () {
754 const cutoff = DateTime.formatIso(
755 DateTime.subtractDuration(yield* DateTime.now, SETTLED_EFFECT_RETENTION),
756 );
757 // Live continuations retain their preceding intent's cumulative crash budget.
758 // Separate batch statements release the write lock between batches.
759 const deleteBatch = sql<{ readonly effect_id: string }>`
760 DELETE FROM orchestration_v2_effect_outbox
761 WHERE rowid IN (
762 SELECT rowid
763 FROM orchestration_v2_effect_outbox
764 WHERE status IN ('succeeded', 'cancelled')
765 AND completed_at < ${cutoff}
766 AND NOT (
767 effect_type = 'provider-runtime.continue'
768 AND EXISTS (
769 SELECT 1 FROM orchestration_v2_projection_runs AS recovery_run
770 WHERE recovery_run.thread_id = orchestration_v2_effect_outbox.thread_id
771 AND recovery_run.status IN ('queued', 'preparing', 'starting', 'running', 'waiting')
772 AND json_extract(recovery_run.payload_json, '$.restartContinuationOfRunId') =
773 json_extract(orchestration_v2_effect_outbox.payload_json, '$.sourceRunId')
774 )
775 )
776 LIMIT ${PRUNE_BATCH_SIZE}
777 )
778 RETURNING effect_id
779 `;
780 let pruned = 0;
781 while (true) {
782 const deleted = (yield* deleteBatch).length;
783 pruned += deleted;
784 if (deleted < PRUNE_BATCH_SIZE) return pruned;
785 yield* Effect.yieldNow;
786 }
787 }).pipe(
788 Effect.mapError((cause) => new EffectOutboxError({ operation: "prune-settled", cause })),
789 ),
790 };
791
792 return service;
793 }),
794 );
795
796 export const layerPruneWorker = Layer.effectDiscard(
797 Effect.gen(function* () {
798 const outbox = yield* EffectOutboxV2;
799 yield* forkParked(
800 outbox.pruneSettled.pipe(
801 Effect.catch((cause) => Effect.logWarning("Failed to prune settled effects", { cause })),
802 Effect.repeat(Schedule.spaced(Duration.hours(1))),
803 ),
804 );
805 }),
806 );