The best way to control Claude Code, Codex and more from any computer or phone

main
ts 1,126 lines 47.7 KB
Raw
1 import { CommandId, type RunId } from "@t3tools/contracts";
2 import * as Cause from "effect/Cause";
3 import * as Context from "effect/Context";
4 import * as DateTime from "effect/DateTime";
5 import * as Duration from "effect/Duration";
6 import * as Effect from "effect/Effect";
7 import * as Exit from "effect/Exit";
8 import * as Fiber from "effect/Fiber";
9 import * as Layer from "effect/Layer";
10 import * as Metric from "effect/Metric";
11 import * as Option from "effect/Option";
12 import * as Schedule from "effect/Schedule";
13 import * as Schema from "effect/Schema";
14 import * as SqlError from "effect/sql/SqlError";
15
16 import {
17 increment,
18 metricAttributes,
19 orchestrationEffectClaimsTotal,
20 orchestrationEffectQueueWait,
21 } from "../observability/Metrics.ts";
22 import * as RunFinalizationService from "./RunFinalizationService.ts";
23 import * as ResourceCleanupService from "./ResourceCleanupService.ts";
24 import * as EffectOutbox from "./EffectOutbox.ts";
25 import * as ProjectionStore from "./ProjectionStore.ts";
26 import * as CheckpointRollbackService from "./CheckpointRollbackService.ts";
27 import * as ProviderSessionManager from "./ProviderSessionManager.ts";
28 import * as ProviderTurnControlService from "./ProviderTurnControlService.ts";
29 import * as ProviderTurnStartService from "./ProviderTurnStartService.ts";
30 import * as RuntimeRequestService from "./RuntimeRequestService.ts";
31 import * as ThreadTitleRegenerationService from "./ThreadTitleRegenerationService.ts";
32 import * as ThreadManagementService from "./ThreadManagementService.ts";
33 import * as ServerSettings from "../serverSettings.ts";
34 import { continueRestartedRun } from "./RestartContinuation.ts";
35
36 const isProviderTurnControlError = Schema.is(ProviderTurnControlService.ProviderTurnControlError);
37
38 export class OrchestrationEffectExecutionError extends Schema.TaggedError<OrchestrationEffectExecutionError>()(
39 "OrchestrationEffectExecutionError",
40 {
41 effectId: Schema.String,
42 effectType: Schema.String,
43 cause: Schema.optional(Schema.Defect()),
44 },
45 ) {}
46
47 const isOrchestrationEffectExecutionError = Schema.is(OrchestrationEffectExecutionError);
48
49 /**
50 * Do not apply this to `provider-turn.restart`: that compound effect also runs
51 * detach and start. Swallowing a start failure that happens to mention
52 * "is not active" would drop the outbox item without ever starting the
53 * replacement turn.
54 */
55 function isNonRetryableProviderTurnControlFailure(effectType: string, errorText: string): boolean {
56 if (effectType !== "provider-turn.interrupt") {
57 return false;
58 }
59 return (
60 /is not active/i.test(errorText) ||
61 /hard teardown is already in progress/i.test(errorText) ||
62 /treating as already interrupted/i.test(errorText) ||
63 /treating as already stopped/i.test(errorText)
64 );
65 }
66
67 // Keep an owner for starting runs when persistence or restart cleanup fails.
68 const ownsStartingRun = (effectType: EffectOutbox.OrchestrationEffectV2["request"]["type"]) =>
69 effectType === "provider-turn.start" || effectType === "provider-turn.restart";
70
71 const cannotSettleRun = (cause: Cause.Cause<unknown>) =>
72 Cause.findErrorOption(cause).pipe(
73 Option.exists(
74 (error) =>
75 isOrchestrationEffectExecutionError(error) &&
76 ProviderTurnStartService.isProviderTurnStartRunStateMissingError(error.cause),
77 ),
78 );
79
80 export interface OrchestrationEffectExecutorV2Shape {
81 readonly execute: (
82 effect: EffectOutbox.OrchestrationEffectV2,
83 options?: { readonly willRetry: boolean },
84 ) => Effect.Effect<void, OrchestrationEffectExecutionError>;
85 }
86
87 export class OrchestrationEffectExecutorV2 extends Context.Service<
88 OrchestrationEffectExecutorV2,
89 OrchestrationEffectExecutorV2Shape
90 >()("t3/orchestration-v2/EffectWorker/OrchestrationEffectExecutorV2") {}
91
92 export const layerExecutor: Layer.Layer<
93 OrchestrationEffectExecutorV2,
94 never,
95 | ProviderSessionManager.ProviderSessionManagerV2
96 | RunFinalizationService.RunFinalizationService
97 | CheckpointRollbackService.CheckpointRollbackServiceV2
98 | ProviderTurnControlService.ProviderTurnControlServiceV2
99 | ProviderTurnStartService.ProviderTurnStartServiceV2
100 | RuntimeRequestService.RuntimeRequestServiceV2
101 | ThreadTitleRegenerationService.ThreadTitleRegenerationService
102 | ThreadManagementService.ThreadManagementService
103 | ProjectionStore.ProjectionStoreV2
104 | ServerSettings.ServerSettingsService
105 > = Layer.effect(
106 OrchestrationEffectExecutorV2,
107 Effect.gen(function* () {
108 const runFinalization = yield* RunFinalizationService.RunFinalizationService;
109 const resourceCleanup = yield* ResourceCleanupService.ResourceCleanupService;
110 const checkpointRollback = yield* CheckpointRollbackService.CheckpointRollbackServiceV2;
111 const providerSessions = yield* ProviderSessionManager.ProviderSessionManagerV2;
112 const providerTurnControl = yield* ProviderTurnControlService.ProviderTurnControlServiceV2;
113 const providerTurnStart = yield* ProviderTurnStartService.ProviderTurnStartServiceV2;
114 const runtimeRequests = yield* RuntimeRequestService.RuntimeRequestServiceV2;
115 const threadTitleRegeneration =
116 yield* ThreadTitleRegenerationService.ThreadTitleRegenerationService;
117 const threads = yield* ThreadManagementService.ThreadManagementService;
118 const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
119 const settings = yield* ServerSettings.ServerSettingsService;
120 const startRun = (
121 effect: EffectOutbox.OrchestrationEffectV2,
122 runId: RunId,
123 willRetry: boolean,
124 ) =>
125 providerTurnStart.start({ threadId: effect.threadId, runId, willRetry }).pipe(
126 Effect.catchCauseIf(
127 (cause) => !willRetry && !Cause.hasInterruptsOnly(cause),
128 (cause) =>
129 Effect.logWarning(
130 "Last run start attempt failed",
131 { effectId: effect.id, effectType: effect.request.type, runId },
132 cause,
133 ).pipe(
134 Effect.andThen(
135 Effect.suspend(() => {
136 const failure = Option.getOrUndefined(Cause.findErrorOption(cause));
137 // A failed read cannot identify the attempt. Keep retry ownership
138 // rather than settling whichever attempt happens to be current.
139 if (
140 !ProviderTurnStartService.isProviderTurnStartError(failure) ||
141 failure.attemptId === undefined ||
142 failure.rootNodeId === undefined
143 ) {
144 return Effect.failCause(cause);
145 }
146 return providerTurnStart.failStartingRun({
147 threadId: effect.threadId,
148 runId,
149 attemptId: failure.attemptId,
150 rootNodeId: failure.rootNodeId,
151 });
152 }),
153 ),
154 ),
155 ),
156 );
157 return OrchestrationEffectExecutorV2.of({
158 execute: Effect.fn("OrchestrationEffectExecutorV2.execute")(function (effect, options) {
159 const willRetry = options?.willRetry ?? false;
160 switch (effect.request.type) {
161 case "provider-runtime.continue": {
162 const sourceRunId = effect.request.sourceRunId;
163 return continueRestartedRun({
164 threadId: effect.threadId,
165 sourceRunId,
166 ...(effect.request.processLossCount === undefined
167 ? {}
168 : { processLossCount: effect.request.processLossCount }),
169 }).pipe(
170 Effect.provideService(ThreadManagementService.ThreadManagementService, threads),
171 Effect.provideService(ServerSettings.ServerSettingsService, settings),
172 Effect.tapError(() =>
173 willRetry
174 ? Effect.void
175 : threads.recoverDelegatedTask(effect.threadId, { sourceRunId }),
176 ),
177 Effect.mapError(
178 (cause) =>
179 new OrchestrationEffectExecutionError({
180 effectId: effect.id,
181 effectType: effect.request.type,
182 cause,
183 }),
184 ),
185 );
186 }
187 case "provider-session.detach":
188 return providerSessions
189 .detach({
190 providerSessionId: effect.request.providerSessionId,
191 threadId: effect.threadId,
192 ...(effect.request.detail === undefined ? {} : { detail: effect.request.detail }),
193 ...(effect.request.revokeMcpCredential === undefined
194 ? {}
195 : { revokeMcpCredential: effect.request.revokeMcpCredential }),
196 })
197 .pipe(
198 Effect.mapError(
199 (cause) =>
200 new OrchestrationEffectExecutionError({
201 effectId: effect.id,
202 effectType: effect.request.type,
203 cause,
204 }),
205 ),
206 );
207 case "provider-session.apply-model-selection":
208 return providerTurnControl
209 .applyModelSelection({
210 threadId: effect.threadId,
211 providerSessionId: effect.request.providerSessionId,
212 providerThreadId: effect.request.providerThreadId,
213 modelSelection: effect.request.modelSelection,
214 })
215 .pipe(
216 Effect.mapError(
217 (cause) =>
218 new OrchestrationEffectExecutionError({
219 effectId: effect.id,
220 effectType: effect.request.type,
221 cause,
222 }),
223 ),
224 );
225 case "provider-session.apply-thread-title":
226 return providerTurnControl
227 .applyThreadTitle({ threadId: effect.threadId, title: effect.request.title })
228 .pipe(
229 Effect.mapError(
230 (cause) =>
231 new OrchestrationEffectExecutionError({
232 effectId: effect.id,
233 effectType: effect.request.type,
234 cause,
235 }),
236 ),
237 );
238 case "provider-turn.start":
239 return startRun(effect, effect.request.runId, willRetry).pipe(
240 Effect.mapError(
241 (cause) =>
242 new OrchestrationEffectExecutionError({
243 effectId: effect.id,
244 effectType: effect.request.type,
245 cause,
246 }),
247 ),
248 );
249 case "provider-turn.interrupt": {
250 const request = effect.request;
251 return providerTurnControl
252 .interrupt({
253 threadId: effect.threadId,
254 providerSessionId: request.providerSessionId,
255 providerThreadId: request.providerThreadId,
256 providerTurnId: request.providerTurnId,
257 ...(request.subagent === undefined
258 ? {}
259 : {
260 subagent: {
261 ...request.subagent,
262 requestedAt: DateTime.makeUnsafe(effect.createdAt),
263 },
264 }),
265 })
266 .pipe(
267 Effect.tapError((cause) =>
268 request.subagent === undefined
269 ? Effect.void
270 : Effect.logWarning("orchestration-v2.subagent-stop-failed", {
271 effectId: effect.id,
272 cause,
273 }),
274 ),
275 Effect.map((outcome) => ({ unresponsive: false, outcome }) as const),
276 // Settling here would end the run while the agent may still be running; the
277 // user decides with Force stop instead.
278 Effect.catchIf(
279 (cause) => request.subagent === undefined && cause.unresponsive === true,
280 () => Effect.succeed({ unresponsive: true, outcome: undefined } as const),
281 ),
282 Effect.catchIf(
283 (cause) =>
284 request.subagent === undefined &&
285 isNonRetryableProviderTurnControlFailure(
286 request.type,
287 Cause.pretty(Cause.fail(cause)),
288 ),
289 () => Effect.succeed({ unresponsive: false, outcome: undefined } as const),
290 ),
291 Effect.andThen(({ unresponsive, outcome }) =>
292 unresponsive
293 ? threads.dispatch({
294 type: "thread.interrupt.unresponsive",
295 commandId: CommandId.make(`${effect.id}:unresponsive`),
296 threadId: effect.threadId,
297 providerThreadId: request.providerThreadId,
298 providerTurnId: request.providerTurnId,
299 })
300 : request.subagent !== undefined && outcome === undefined
301 ? Effect.void
302 : threads.dispatch({
303 type: "thread.background-work.settle",
304 commandId: CommandId.make(`${effect.id}:background-work-settled`),
305 ...(request.subagent === undefined || outcome === undefined
306 ? {}
307 : {
308 subagent: {
309 id: request.subagent.id,
310 stoppedAt: DateTime.makeUnsafe(effect.createdAt),
311 outcome,
312 },
313 }),
314 threadId: effect.threadId,
315 providerThreadId: request.providerThreadId,
316 providerTurnId: request.providerTurnId,
317 }),
318 ),
319 Effect.mapError(
320 (cause) =>
321 new OrchestrationEffectExecutionError({
322 effectId: effect.id,
323 effectType: request.type,
324 cause,
325 }),
326 ),
327 );
328 }
329 case "provider-turn.steer":
330 return providerTurnControl
331 .steer({
332 threadId: effect.threadId,
333 providerSessionId: effect.request.providerSessionId,
334 providerThreadId: effect.request.providerThreadId,
335 providerTurnId: effect.request.providerTurnId,
336 messageId: effect.request.messageId,
337 ...(effect.request.modelSelection === undefined
338 ? {}
339 : { modelSelection: effect.request.modelSelection }),
340 })
341 .pipe(
342 Effect.tap(() =>
343 Effect.gen(function* () {
344 if (effect.request.type !== "provider-turn.steer") return;
345 const messageId = effect.request.messageId;
346 const projection = yield* threads.getThreadRecords(
347 effect.threadId,
348 ["messages", "runs"],
349 { messageIds: [effect.request.messageId] },
350 );
351 const message = projection.messages.find((row) => row.id === messageId);
352 if (message?.delegatedCompletion === undefined) return;
353 yield* threads.dispatch({
354 type: "notification.delivery.accept",
355 commandId: CommandId.make(`command:mailbox-accepted:${effect.id}`),
356 threadId: effect.threadId,
357 messageId: message.id,
358 });
359 }),
360 ),
361 Effect.catch((error) =>
362 Effect.gen(function* () {
363 if (
364 !isProviderTurnControlError(error) ||
365 (!error.turnCompleted && !error.steerDeclined) ||
366 effect.request.type !== "provider-turn.steer"
367 ) {
368 return yield* error;
369 }
370 const projection = yield* threads.getThreadRecords(
371 effect.threadId,
372 ["messages", "runs"],
373 { messageIds: [effect.request.messageId] },
374 );
375 const messageId = effect.request.messageId;
376 const message = projection.messages.find((item) => item.id === messageId);
377 const run = projection.runs.find((item) => item.id === message?.runId);
378 if (message === undefined || run === undefined) return yield* error;
379 // Notifications must be queued messages.
380 const queued =
381 message.delegatedCompletion !== undefined ||
382 message.notification !== undefined ||
383 error.steerDeclined === true;
384 // Reuse message ids and receipts so retries cannot duplicate messages or follow-ups.
385 yield* threads
386 .dispatch({
387 type: "message.dispatch",
388 commandId: CommandId.make(`command:steer-follow-up:${effect.id}`),
389 threadId: effect.threadId,
390 messageId: message.id,
391 ...(error.steerDeclined ? { steerFollowUpOfRunId: run.id } : {}),
392 ...(error.steerDeclined &&
393 effect.request.steerFollowUpAfterRunOrdinal !== undefined
394 ? {
395 steerFollowUpAfterRunOrdinal:
396 effect.request.steerFollowUpAfterRunOrdinal,
397 }
398 : {}),
399 text: message.text,
400 ...(message.context ? { context: message.context } : {}),
401 attachments: message.attachments,
402 ...(error.steerDeclined && effect.request.modelSelection !== undefined
403 ? { modelSelection: effect.request.modelSelection }
404 : message.delegatedCompletion === undefined
405 ? {}
406 : { modelSelection: run.modelSelection }),
407 dispatchMode: {
408 type: queued ? "queue_after_active" : "start_immediately",
409 },
410 createdBy: message.createdBy,
411 creationSource: message.creationSource,
412 ...(message.delegatedCompletion === undefined
413 ? {}
414 : { delegatedCompletion: message.delegatedCompletion }),
415 ...(message.notification === undefined
416 ? {}
417 : { notification: message.notification }),
418 ...(message.scheduledTaskId === undefined
419 ? {}
420 : { scheduledTaskId: message.scheduledTaskId }),
421 ...(message.senderThreadId === undefined
422 ? {}
423 : { senderThreadId: message.senderThreadId }),
424 ...(message.linkedSender === undefined
425 ? {}
426 : { linkedSender: message.linkedSender }),
427 })
428 .pipe(
429 // A disabled instance cannot retry this completed turn; its result stands.
430 Effect.catchTags({
431 ThreadManagementProviderInstanceDisabledError: (disabledError) =>
432 Effect.logInfo(
433 "orchestration-v2.effect-worker.steer-follow-up-disabled",
434 {
435 threadId: effect.threadId,
436 messageId: message.id,
437 projectId: disabledError.projectId,
438 instanceId: disabledError.instanceId,
439 },
440 ),
441 }),
442 );
443 }),
444 ),
445 Effect.mapError(
446 (cause) =>
447 new OrchestrationEffectExecutionError({
448 effectId: effect.id,
449 effectType: effect.request.type,
450 cause,
451 }),
452 ),
453 );
454 case "provider-turn.restart":
455 return providerTurnControl
456 .interruptAndAwaitTerminal({
457 threadId: effect.threadId,
458 providerSessionId: effect.request.providerSessionId,
459 providerThreadId: effect.request.providerThreadId,
460 providerTurnId: effect.request.providerTurnId,
461 interruptedAttemptId: effect.request.interruptedAttemptId,
462 ...(effect.request.sessionTransition?.type === "replace"
463 ? {
464 replacementProviderSessionId:
465 effect.request.sessionTransition.replacementProviderSessionId,
466 }
467 : {}),
468 })
469 .pipe(
470 Effect.andThen(
471 effect.request.sessionTransition?.type === "replace"
472 ? providerSessions.detach({
473 providerSessionId: effect.request.providerSessionId,
474 threadId: effect.threadId,
475 detail: "Selection change requires a provider session restart.",
476 })
477 : effect.request.sessionTransition?.type === "detach"
478 ? providerSessions.detach({
479 providerSessionId: effect.request.providerSessionId,
480 threadId: effect.threadId,
481 detail: "Provider thread handoff replaced this session binding.",
482 })
483 : Effect.void,
484 ),
485 Effect.andThen(
486 // Failed interruption or detach must not report the old process stopped.
487 startRun(effect, effect.request.runId, willRetry),
488 ),
489 Effect.mapError(
490 (cause) =>
491 new OrchestrationEffectExecutionError({
492 effectId: effect.id,
493 effectType: effect.request.type,
494 cause,
495 }),
496 ),
497 );
498 case "runtime-request.respond": {
499 const requestId = effect.request.requestId;
500 return runtimeRequests
501 .respond({
502 threadId: effect.threadId,
503 providerSessionId: effect.request.providerSessionId,
504 requestId: effect.request.requestId,
505 ...(effect.request.decision === undefined
506 ? {}
507 : { decision: effect.request.decision }),
508 ...(effect.request.feedback === undefined
509 ? {}
510 : { feedback: effect.request.feedback }),
511 ...(effect.request.answers === undefined
512 ? {}
513 : { answers: effect.request.answers }),
514 })
515 .pipe(
516 Effect.tapCause((cause) =>
517 willRetry || Cause.hasInterruptsOnly(cause)
518 ? Effect.void
519 : threads
520 .dispatch({
521 type: "runtime-request.delivery.fail",
522 commandId: CommandId.make(`${effect.commandId}:delivery-failed`),
523 threadId: effect.threadId,
524 requestId,
525 })
526 .pipe(
527 Effect.catchCause((recordCause) =>
528 Effect.logWarning("Failed to record answer delivery failure", {
529 effectId: effect.id,
530 cause: recordCause,
531 }),
532 ),
533 ),
534 ),
535 Effect.mapError(
536 (cause) =>
537 new OrchestrationEffectExecutionError({
538 effectId: effect.id,
539 effectType: effect.request.type,
540 cause,
541 }),
542 ),
543 );
544 }
545 case "provider-thread.rollback":
546 return checkpointRollback
547 .execute({
548 threadId: effect.threadId,
549 providerThreadId: effect.request.providerThreadId,
550 checkpointId: effect.request.checkpointId,
551 scopeId: effect.request.scopeId,
552 ...(effect.request.restoreFiles === undefined
553 ? {}
554 : { restoreFiles: effect.request.restoreFiles }),
555 })
556 .pipe(
557 Effect.tapCause((cause) =>
558 willRetry || Cause.hasInterruptsOnly(cause)
559 ? Effect.void
560 : threads
561 .dispatch({
562 type: "checkpoint.rollback.fail",
563 commandId: CommandId.make(`${effect.commandId}:rollback-failed`),
564 threadId: effect.threadId,
565 requestId: effect.commandId,
566 message: Option.match(Cause.findErrorOption(cause), {
567 onNone: () => CheckpointRollbackService.ROLLBACK_FAILED_MESSAGE,
568 onSome: CheckpointRollbackService.rollbackFailureClientMessage,
569 }),
570 })
571 .pipe(
572 Effect.catchCause((recordCause) =>
573 Effect.logWarning("Failed to record rollback failure", {
574 effectId: effect.id,
575 cause: recordCause,
576 }),
577 ),
578 ),
579 ),
580 Effect.mapError(
581 (cause) =>
582 new OrchestrationEffectExecutionError({
583 effectId: effect.id,
584 effectType: effect.request.type,
585 cause,
586 }),
587 ),
588 );
589 case "checkpoint.capture":
590 return runFinalization
591 .finalize({
592 threadId: effect.threadId,
593 runId: effect.request.runId,
594 scopeId: effect.request.scopeId,
595 })
596 .pipe(
597 // Completion bookkeeping must survive a missed live run update, including capture retries.
598 Effect.andThen(
599 threads.recoverDelegatedTask(effect.threadId, {
600 completedRunId: effect.request.runId,
601 }),
602 ),
603 Effect.mapError(
604 (cause) =>
605 new OrchestrationEffectExecutionError({
606 effectId: effect.id,
607 effectType: effect.request.type,
608 cause,
609 }),
610 ),
611 );
612 case "scheduled-tasks.cleanup":
613 return resourceCleanup.cleanupScheduledTasks(effect.threadId).pipe(
614 Effect.mapError(
615 (cause) =>
616 new OrchestrationEffectExecutionError({
617 effectId: effect.id,
618 effectType: effect.request.type,
619 cause,
620 }),
621 ),
622 );
623 // Archive and delete queue this one per-thread teardown, so it also releases the
624 // thread's device claims; the devices stay booted for other threads.
625 case "terminal.cleanup":
626 return Effect.partition(
627 [
628 resourceCleanup.cleanupTerminals(effect.threadId),
629 resourceCleanup.cleanupDevices(effect.threadId),
630 ],
631 (cleanup) => cleanup,
632 ).pipe(
633 Effect.flatMap(([, [failure]]) =>
634 failure === undefined ? Effect.void : Effect.fail(failure),
635 ),
636 Effect.mapError(
637 (cause) =>
638 new OrchestrationEffectExecutionError({
639 effectId: effect.id,
640 effectType: effect.request.type,
641 cause,
642 }),
643 ),
644 );
645 case "preview.cleanup":
646 return resourceCleanup.cleanupPreviews(effect.threadId).pipe(
647 Effect.mapError(
648 (cause) =>
649 new OrchestrationEffectExecutionError({
650 effectId: effect.id,
651 effectType: effect.request.type,
652 cause,
653 }),
654 ),
655 );
656 case "attachment.cleanup": {
657 const request = effect.request;
658 return Effect.gen(function* () {
659 // Another message or answer may share the file, or a stale queued edit may restore it.
660 const referenced = new Set(
661 request.skipReferenced
662 ? yield* projectionStore.getThreadAttachmentIds(effect.threadId, {
663 includeShownByLiveFork: true,
664 })
665 : [],
666 );
667 yield* resourceCleanup.cleanupAttachments(
668 request.attachmentIds.filter((id) => !referenced.has(id)),
669 );
670 }).pipe(
671 Effect.mapError(
672 (cause) =>
673 new OrchestrationEffectExecutionError({
674 effectId: effect.id,
675 effectType: effect.request.type,
676 cause,
677 }),
678 ),
679 );
680 }
681 case "thread-title.generate":
682 return threadTitleRegeneration
683 .execute({
684 threadId: effect.threadId,
685 requestId: effect.commandId,
686 kind: effect.request.kind,
687 })
688 .pipe(
689 Effect.mapError(
690 (cause) =>
691 new OrchestrationEffectExecutionError({
692 effectId: effect.id,
693 effectType: effect.request.type,
694 cause,
695 }),
696 ),
697 );
698 case "delegated-tasks.stop":
699 return threads
700 .stopDelegatedTasks({
701 threadId: effect.threadId,
702 commandId: effect.commandId,
703 reason: effect.request.reason,
704 })
705 .pipe(
706 Effect.mapError(
707 (cause) =>
708 new OrchestrationEffectExecutionError({
709 effectId: effect.id,
710 effectType: effect.request.type,
711 cause,
712 }),
713 ),
714 );
715 }
716 }),
717 });
718 }),
719 );
720
721 // A running effect blocks later work on its thread and its lease is never reclaimed in-process,
722 // so a stop that hangs must give up and fall through to the normal retry/fail path.
723 const DELEGATED_TASKS_STOP_TIMEOUT = Duration.seconds(30);
724
725 const boundExecution =
726 (effect: EffectOutbox.OrchestrationEffectV2) =>
727 (execution: Effect.Effect<void, OrchestrationEffectExecutionError>) =>
728 effect.request.type === "delegated-tasks.stop"
729 ? execution.pipe(
730 Effect.timeoutOrElse({
731 duration: DELEGATED_TASKS_STOP_TIMEOUT,
732 orElse: () =>
733 Effect.fail(
734 new OrchestrationEffectExecutionError({
735 effectId: effect.id,
736 effectType: effect.request.type,
737 cause: `Timed out after ${Duration.format(DELEGATED_TASKS_STOP_TIMEOUT)}.`,
738 }),
739 ),
740 }),
741 )
742 : execution;
743
744 export class OrchestrationEffectWorkerError extends Schema.TaggedError<OrchestrationEffectWorkerError>()(
745 "OrchestrationEffectWorkerError",
746 {
747 operation: Schema.String,
748 effectId: Schema.optional(Schema.String),
749 cause: Schema.optional(Schema.Defect()),
750 },
751 ) {}
752
753 const isOrchestrationEffectWorkerError = Schema.is(OrchestrationEffectWorkerError);
754
755 // Only the claiming worker settles its row, and a running row blocks every later effect on its
756 // thread until the server restarts. onError masks interruption, so retries must restore it.
757 const isLockTimeout = (error: EffectOutbox.EffectOutboxError) =>
758 SqlError.isSqlError(error.cause) && error.cause.reason._tag === "LockTimeoutError";
759 const lockRetrySchedule = Schedule.spaced(Duration.millis(100));
760 const retryWhileLocked = <A>(settlement: Effect.Effect<A, EffectOutbox.EffectOutboxError>) =>
761 Effect.interruptible(
762 Effect.retry(settlement, { schedule: lockRetrySchedule, while: isLockTimeout }),
763 );
764
765 type ClaimFilter = Omit<Parameters<EffectOutbox.EffectOutboxV2Shape["claimNext"]>[0], "workerId">;
766
767 export interface OrchestrationEffectWorkerV2Shape {
768 readonly awaitWork: Effect.Effect<void>;
769 readonly runOnce: (input?: {
770 readonly maxRunningTitleGenerations?: number;
771 }) => Effect.Effect<boolean, OrchestrationEffectWorkerError>;
772 readonly runRecoveryOnce: Effect.Effect<boolean, OrchestrationEffectWorkerError>;
773 readonly nextClaimableAt: (input?: {
774 readonly maxRunningTitleGenerations?: number;
775 }) => Effect.Effect<Option.Option<DateTime.Utc>, OrchestrationEffectWorkerError>;
776 readonly drain: (maxEffects?: number) => Effect.Effect<number, OrchestrationEffectWorkerError>;
777 }
778
779 export class OrchestrationEffectWorkerV2 extends Context.Service<
780 OrchestrationEffectWorkerV2,
781 OrchestrationEffectWorkerV2Shape
782 >()("t3/orchestration-v2/EffectWorker/OrchestrationEffectWorkerV2") {}
783
784 export interface OrchestrationEffectWorkerOptions {
785 readonly workerId?: string;
786 readonly maxAttempts?: number;
787 }
788
789 export const layerWithOptions = (
790 options: OrchestrationEffectWorkerOptions = {},
791 ): Layer.Layer<
792 OrchestrationEffectWorkerV2,
793 never,
794 EffectOutbox.EffectOutboxV2 | OrchestrationEffectExecutorV2
795 > =>
796 Layer.effect(
797 OrchestrationEffectWorkerV2,
798 Effect.gen(function* () {
799 const outbox = yield* EffectOutbox.EffectOutboxV2;
800 const executor = yield* OrchestrationEffectExecutorV2;
801 const workerId = options.workerId ?? `orchestration-v2:${process.pid}`;
802 const maxAttempts = Math.max(1, options.maxAttempts ?? 5);
803 const wasCancelled = (effectId: string) =>
804 outbox.get(effectId).pipe(
805 Effect.map(
806 Option.match({
807 onNone: () => false,
808 onSome: (effect) => effect.status === "cancelled",
809 }),
810 ),
811 );
812 const requeueClaim = (
813 effect: EffectOutbox.OrchestrationEffectV2,
814 cause: Cause.Cause<unknown>,
815 ) =>
816 Cause.hasInterruptsOnly(cause)
817 ? Effect.void
818 : outbox
819 .retry({
820 effectId: effect.id,
821 workerId,
822 error: `Worker failed before settling the claimed effect: ${Cause.pretty(cause)}`,
823 delayMs: 0,
824 })
825 .pipe(
826 retryWhileLocked,
827 Effect.flatMap((requeued) =>
828 requeued
829 ? Effect.logWarning("Requeued effect after unexpected worker failure", {
830 effectId: effect.id,
831 effectType: effect.request.type,
832 })
833 : Effect.logWarning("Could not requeue effect after worker lost its lease", {
834 effectId: effect.id,
835 effectType: effect.request.type,
836 }),
837 ),
838 Effect.catchCause((requeueCause) =>
839 Effect.logError("Failed to requeue effect after unexpected worker failure", {
840 effectId: effect.id,
841 effectType: effect.request.type,
842 error: Cause.pretty(requeueCause),
843 }),
844 ),
845 );
846 const terminalizeClaim = (
847 effect: EffectOutbox.OrchestrationEffectV2,
848 cause: Cause.Cause<unknown>,
849 ) => {
850 if (Cause.hasInterruptsOnly(cause)) return Effect.void;
851 return outbox
852 .fail({
853 effectId: effect.id,
854 workerId,
855 error: `Worker failed to settle a process-bound effect after execution started: ${Cause.pretty(cause)}`,
856 })
857 .pipe(
858 retryWhileLocked,
859 Effect.flatMap((failed) =>
860 failed
861 ? Effect.logError("Terminalized process-bound effect after settlement failure", {
862 effectId: effect.id,
863 effectType: effect.request.type,
864 })
865 : Effect.logWarning(
866 "Could not terminalize process-bound effect after worker lost its lease",
867 {
868 effectId: effect.id,
869 effectType: effect.request.type,
870 },
871 ),
872 ),
873 Effect.catchCause((failCause) =>
874 Effect.logError(
875 "Failed to terminalize process-bound effect after settlement failure",
876 {
877 effectId: effect.id,
878 effectType: effect.request.type,
879 error: Cause.pretty(failCause),
880 },
881 ),
882 ),
883 );
884 };
885 const recoverPostSuccessSettlement = (
886 effect: EffectOutbox.OrchestrationEffectV2,
887 cause: Cause.Cause<unknown>,
888 ) =>
889 EffectOutbox.REPLAY_SAFE_EFFECT_TYPES_AFTER_PROCESS_LOSS.some(
890 (effectType) => effectType === effect.request.type,
891 )
892 ? requeueClaim(effect, cause)
893 : terminalizeClaim(effect, cause);
894
895 // The worker probes for work repeatedly, including when the outbox is empty.
896 const runOnce = Effect.fnUntraced(
897 function* (filter: ClaimFilter = {}) {
898 const claimExit = yield* Effect.exit(outbox.claimNext({ workerId, ...filter }));
899 yield* increment(orchestrationEffectClaimsTotal, {
900 result: Exit.isFailure(claimExit)
901 ? "error"
902 : Option.isNone(claimExit.value)
903 ? "empty"
904 : "claimed",
905 });
906 if (Exit.isFailure(claimExit)) return yield* Effect.failCause(claimExit.cause);
907 const claimed = claimExit.value;
908 if (Option.isNone(claimed)) {
909 return false;
910 }
911 const effect = claimed.value;
912 // Arm cancellation before rereading durable state to close the read-to-registration race.
913 const cancellation = outbox
914 .awaitCancellation(effect.id)
915 .pipe(Effect.as("cancelled" as const));
916 const cancelledBeforeExecution = yield* Effect.gen(function* () {
917 const claimedAt = DateTime.toEpochMillis(yield* DateTime.now);
918 const eligibleAt = Math.max(
919 DateTime.toEpochMillis(DateTime.makeUnsafe(effect.createdAt)),
920 DateTime.toEpochMillis(DateTime.makeUnsafe(effect.availableAt)),
921 );
922 yield* Metric.update(
923 Metric.withAttributes(
924 orchestrationEffectQueueWait,
925 metricAttributes({ effect_type: effect.request.type }),
926 ),
927 Duration.millis(Math.max(0, claimedAt - eligibleAt)),
928 );
929 // Recheck durable cancellation after registering the Deferred; cancellation may have committed after claim.
930 if (yield* wasCancelled(effect.id)) {
931 yield* outbox.clearCancellation(effect.id);
932 return true;
933 }
934 return false;
935 }).pipe(Effect.onError((cause) => requeueClaim(effect, cause)));
936 if (cancelledBeforeExecution) return true;
937
938 const execution = executor
939 .execute(effect, { willRetry: effect.attemptCount < maxAttempts })
940 .pipe(boundExecution(effect), Effect.as("executed" as const));
941 const exit = yield* Effect.exit(Effect.raceFirst(execution, cancellation)).pipe(
942 Effect.ensuring(outbox.clearCancellation(effect.id)),
943 );
944 if (Exit.isSuccess(exit) && exit.value === "cancelled") {
945 return true;
946 }
947 if (Exit.isSuccess(exit)) {
948 return yield* Effect.gen(function* () {
949 const completed = yield* outbox.succeed({ effectId: effect.id, workerId });
950 if (!completed) {
951 if (yield* wasCancelled(effect.id)) return true;
952 return yield* new OrchestrationEffectWorkerError({
953 operation: "complete",
954 effectId: effect.id,
955 cause: "The worker no longer owns the effect lease.",
956 });
957 }
958 return true;
959 }).pipe(Effect.onError((cause) => recoverPostSuccessSettlement(effect, cause)));
960 }
961
962 const error = Cause.pretty(exit.cause);
963 const nonRetryable = isNonRetryableProviderTurnControlFailure(effect.request.type, error);
964 yield* Effect.logWarning("Orchestration effect execution failed", {
965 effectId: effect.id,
966 effectType: effect.request.type,
967 attemptCount: effect.attemptCount,
968 nonRetryable,
969 error,
970 });
971 const updated = nonRetryable
972 ? yield* outbox
973 .succeed({ effectId: effect.id, workerId })
974 .pipe(Effect.onError((cause) => terminalizeClaim(effect, cause)))
975 : effect.attemptCount >= maxAttempts &&
976 (!ownsStartingRun(effect.request.type) || cannotSettleRun(exit.cause))
977 ? yield* outbox
978 .fail({ effectId: effect.id, workerId, error })
979 .pipe(Effect.onError((cause) => terminalizeClaim(effect, cause)))
980 : yield* outbox
981 .retry({
982 effectId: effect.id,
983 workerId,
984 error,
985 delayMs: Math.min(30_000, 100 * 2 ** Math.max(0, effect.attemptCount - 1)),
986 })
987 .pipe(Effect.onError((cause) => requeueClaim(effect, cause)));
988 if (!updated) {
989 if (yield* wasCancelled(effect.id)) return true;
990 return yield* new OrchestrationEffectWorkerError({
991 operation: "reschedule",
992 effectId: effect.id,
993 cause: "The worker no longer owns the effect lease.",
994 });
995 }
996 return true;
997 },
998 (effect, _filter?: ClaimFilter) =>
999 effect.pipe(
1000 Effect.mapError((cause) =>
1001 isOrchestrationEffectWorkerError(cause)
1002 ? cause
1003 : new OrchestrationEffectWorkerError({ operation: "run", cause }),
1004 ),
1005 ),
1006 );
1007
1008 return OrchestrationEffectWorkerV2.of({
1009 awaitWork: outbox.awaitAvailable,
1010 runOnce,
1011 runRecoveryOnce: runOnce({ excludeRestartContinuations: true }),
1012 // Idle workers poll this; a span per poll would bury useful traces.
1013 nextClaimableAt: Effect.fnUntraced(
1014 function* (input) {
1015 return yield* outbox.nextClaimableAt(input);
1016 },
1017 Effect.mapError(
1018 (cause) =>
1019 new OrchestrationEffectWorkerError({
1020 operation: "next-claimable",
1021 cause,
1022 }),
1023 ),
1024 ),
1025 drain: Effect.fn("EffectWorker.drain")(function* (maxEffects = Number.MAX_SAFE_INTEGER) {
1026 let completed = 0;
1027 while (completed < maxEffects && (yield* runOnce())) {
1028 completed += 1;
1029 }
1030 return completed;
1031 }),
1032 });
1033 }),
1034 );
1035
1036 export const layer = layerWithOptions();
1037
1038 export interface OrchestrationEffectDaemonOptions {
1039 readonly concurrency?: number;
1040 readonly livenessPollIntervalMs?: number;
1041 }
1042
1043 const DEFAULT_EFFECT_WORKER_CONCURRENCY = 4;
1044 const DEFAULT_EFFECT_WORKER_LIVENESS_POLL_INTERVAL_MS = 30_000;
1045 // A provider request can go unanswered indefinitely. After this wait the slot claims other work
1046 // while the execution keeps its running row (which blocks its own thread) and settles it later.
1047 const EFFECT_WORKER_SLOT_WAIT = Duration.seconds(30);
1048
1049 export const runDaemonWithOptions = (options: OrchestrationEffectDaemonOptions = {}) =>
1050 Effect.scoped(
1051 Effect.gen(function* () {
1052 const worker = yield* OrchestrationEffectWorkerV2;
1053 const scope = yield* Effect.scope;
1054 const requestedConcurrency = options.concurrency ?? DEFAULT_EFFECT_WORKER_CONCURRENCY;
1055 const concurrency = Number.isFinite(requestedConcurrency)
1056 ? Math.max(1, Math.floor(requestedConcurrency))
1057 : DEFAULT_EFFECT_WORKER_CONCURRENCY;
1058 const requestedLivenessPollIntervalMs =
1059 options.livenessPollIntervalMs ?? DEFAULT_EFFECT_WORKER_LIVENESS_POLL_INTERVAL_MS;
1060 const livenessPollIntervalMs = Number.isFinite(requestedLivenessPollIntervalMs)
1061 ? Math.max(1, Math.floor(requestedLivenessPollIntervalMs))
1062 : DEFAULT_EFFECT_WORKER_LIVENESS_POLL_INTERVAL_MS;
1063 // Title generation can run for minutes; leave at least one worker free for turn work.
1064 const claimLimits = concurrency > 1 ? { maxRunningTitleGenerations: concurrency - 1 } : {};
1065 // Notifications wake promptly; availableAt persists retry timing, and liveness polling recovers missed wakeups.
1066 const runWorker = Effect.gen(function* () {
1067 while (true) {
1068 const attempt = yield* worker.runOnce(claimLimits).pipe(
1069 Effect.map((worked) => (worked ? ("worked" as const) : ("idle" as const))),
1070 Effect.catchCause((cause) =>
1071 Effect.logWarning("Orchestration effect worker failed", cause).pipe(
1072 Effect.as("failed" as const),
1073 ),
1074 ),
1075 Effect.forkIn(scope),
1076 );
1077 const outcome = yield* Fiber.join(attempt).pipe(
1078 Effect.timeoutOption(EFFECT_WORKER_SLOT_WAIT),
1079 Effect.map(Option.getOrElse(() => "worked" as const)),
1080 );
1081 if (outcome === "worked") {
1082 yield* Effect.yieldNow;
1083 continue;
1084 }
1085 if (outcome === "failed") {
1086 // Failed claims can leave overdue rows; cool off instead of rescheduling at the one-millisecond floor.
1087 yield* Effect.sleep(Duration.millis(Math.min(1_000, livenessPollIntervalMs)));
1088 continue;
1089 }
1090
1091 const nextClaimableAt = yield* worker
1092 .nextClaimableAt(claimLimits)
1093 .pipe(
1094 Effect.catchCause((cause) =>
1095 Effect.logWarning(
1096 "Failed to read the next orchestration effect deadline",
1097 cause,
1098 ).pipe(Effect.as(Option.none<DateTime.Utc>())),
1099 ),
1100 );
1101 const now = DateTime.toEpochMillis(yield* DateTime.now);
1102 const sleepMs = Option.match(nextClaimableAt, {
1103 onNone: () => livenessPollIntervalMs,
1104 onSome: (availableAt) => {
1105 const untilAvailable = DateTime.toEpochMillis(availableAt) - now;
1106 return Math.min(livenessPollIntervalMs, untilAvailable > 0 ? untilAvailable : 25);
1107 },
1108 });
1109 yield* Effect.raceFirst(
1110 worker.awaitWork.pipe(Effect.as("notified" as const)),
1111 Effect.sleep(Duration.millis(sleepMs)).pipe(Effect.as("scheduled" as const)),
1112 );
1113 }
1114 });
1115
1116 return yield* Effect.all(
1117 Array.from({ length: concurrency }, () => runWorker),
1118 {
1119 concurrency: "unbounded",
1120 discard: true,
1121 },
1122 );
1123 }),
1124 );
1125
1126 export const runDaemon = runDaemonWithOptions();