The best way to control Claude Code, Codex and more from any computer or phone

main
ts 350 lines 14.3 KB
Raw
1 import {
2 CommandId,
3 type MessageId,
4 type OrchestrationV2ThreadProjection,
5 type ThreadId,
6 } from "@t3tools/contracts";
7 import * as Effect from "effect/Effect";
8 import * as Layer from "effect/Layer";
9 import * as Option from "effect/Option";
10 import * as Ref from "effect/Ref";
11 import * as Stream from "effect/Stream";
12
13 import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
14 import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests";
15 import * as KeyedLock from "@t3tools/shared/KeyedLock";
16 import type * as Orchestrator from "./Orchestrator.ts";
17 import * as ThreadManagementService from "./ThreadManagementService.ts";
18 import { isUndeliveredMailboxSteer } from "./NotificationMailbox.ts";
19
20 const CONTINUATION_MESSAGE_TEXT = "Background task completed.";
21
22 function delegatedCompletionText(taskIds: ReadonlyArray<string>): string {
23 const taskList = taskIds.join(", ");
24 return taskIds.length === 1
25 ? `Delegated task ${taskList} reached a terminal state. Use task_status with taskId ${taskList} to read the result.`
26 : `Delegated tasks ${taskList} reached terminal states. Use task_status with each taskId to read the results.`;
27 }
28
29 function currentDelegatedCompletionDelivery(
30 projection: Pick<OrchestrationV2ThreadProjection, "messages" | "runs" | "providerTurns">,
31 completion: NonNullable<
32 ProviderContinuationRequests.ProviderContinuationRequest["delegatedCompletion"]
33 >,
34 ) {
35 const sourceRun = projection.runs.find((candidate) => candidate.id === completion.parentRunId);
36 const delivery = sourceRun?.delegatedCompletion?.delivery;
37 const alreadyDispatched = projection.messages.some(
38 (message) => message.id === completion.messageId,
39 );
40 if (
41 sourceRun?.delegatedCompletion?.disposition !== "open" ||
42 delivery === null ||
43 delivery === undefined ||
44 delivery.generation !== completion.generation ||
45 delivery.messageId !== completion.messageId ||
46 (alreadyDispatched && !isUndeliveredMailboxSteer(projection, completion.messageId))
47 ) {
48 return undefined;
49 }
50 return delivery;
51 }
52
53 function delegatedCompletionRetryKey(
54 request: ProviderContinuationRequests.ProviderContinuationRequest,
55 completion: NonNullable<
56 ProviderContinuationRequests.ProviderContinuationRequest["delegatedCompletion"]
57 >,
58 ): string {
59 return `${request.threadId}:${completion.parentRunId}:${completion.generation}:${completion.messageId}`;
60 }
61
62 export const layer = Layer.effectDiscard(
63 Effect.gen(function* () {
64 const ids = yield* IdAllocator.IdAllocatorV2;
65 const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests;
66 const threads = yield* ThreadManagementService.ThreadManagementService;
67 const retryAttempts = yield* Ref.make(new Map<string, number>());
68 const continuations = yield* KeyedLock.makeRunner<ThreadId>();
69
70 const clearRetryAttempt = (key: string) =>
71 Ref.update(retryAttempts, (current) => {
72 if (!current.has(key)) return current;
73 const updated = new Map(current);
74 updated.delete(key);
75 return updated;
76 });
77
78 const nextRetryDelay = (key: string) =>
79 Ref.modify(retryAttempts, (current) => {
80 const attempt = current.get(key) ?? 0;
81 const updated = new Map(current);
82 updated.set(key, attempt + 1);
83 return [Math.min(100 * 2 ** Math.min(attempt, 6), 5_000), updated] as const;
84 });
85
86 // Cancelling or stopping a queued continuation ends it unstarted; only the request can release the adapter's wake latch.
87 const releaseIfNeverStarted = Effect.fn("ProviderContinuationService.releaseIfNeverStarted")(
88 function* (
89 request: ProviderContinuationRequests.ProviderContinuationRequest,
90 messageId: MessageId,
91 dispatched: Orchestrator.OrchestratorV2DispatchResult,
92 ) {
93 const onRunEndedWithoutStarting = request.onRunEndedWithoutStarting;
94 if (onRunEndedWithoutStarting === undefined) return;
95 const runId = dispatched.storedEvents.flatMap(({ event }) =>
96 event.type === "run.created" && event.payload.userMessageId === messageId
97 ? [event.payload.id]
98 : [],
99 )[0];
100 if (runId === undefined) return;
101 yield* threads
102 .streamStoredEventsFrom({
103 threadId: request.threadId,
104 afterSequence: dispatched.sequence,
105 eventType: "run.updated",
106 })
107 .pipe(
108 Stream.filter(
109 ({ event }) =>
110 event.type === "run.updated" &&
111 event.payload.id === runId &&
112 ThreadManagementService.isTerminalRunStatus(event.payload.status),
113 ),
114 Stream.runHead,
115 Effect.flatMap((ended) =>
116 Option.isSome(ended) &&
117 ended.value.event.type === "run.updated" &&
118 ended.value.event.payload.startedAt === null
119 ? onRunEndedWithoutStarting()
120 : Effect.void,
121 ),
122 Effect.catchCause((cause) =>
123 Effect.logWarning("orchestration-v2.provider-continuation.run-watch-failed", {
124 threadId: request.threadId,
125 providerThreadId: request.providerThreadId,
126 cause,
127 }),
128 ),
129 Effect.forkScoped,
130 );
131 },
132 );
133
134 const dispatchContinuation = Effect.fn("ProviderContinuationService.dispatchContinuation")(
135 function* (request: ProviderContinuationRequests.ProviderContinuationRequest) {
136 const projection = yield* threads.getThreadRecords(
137 request.threadId,
138 ["messages", "runs", "providerTurns"],
139 {
140 messageIds:
141 request.delegatedCompletion === undefined
142 ? []
143 : [request.delegatedCompletion.messageId],
144 },
145 );
146 if (projection.thread.archivedAt !== null || projection.thread.deletedAt !== null) {
147 yield* Effect.logInfo("orchestration-v2.provider-continuation.thread-archived", {
148 threadId: request.threadId,
149 providerThreadId: request.providerThreadId,
150 });
151 if (request.delegatedCompletion !== undefined) {
152 yield* clearRetryAttempt(
153 delegatedCompletionRetryKey(request, request.delegatedCompletion),
154 );
155 }
156 if (request.clearIfCurrent !== undefined) {
157 yield* request.clearIfCurrent();
158 } else if (request.dispatchIfCurrent !== undefined) {
159 yield* request.dispatchIfCurrent(Effect.void);
160 }
161 return;
162 }
163 if (request.delegatedCompletion !== undefined) {
164 const retryKey = delegatedCompletionRetryKey(request, request.delegatedCompletion);
165 const delivery = currentDelegatedCompletionDelivery(
166 projection,
167 request.delegatedCompletion,
168 );
169 if (delivery === undefined) {
170 yield* clearRetryAttempt(retryKey);
171 return;
172 }
173 const commandId = yield* ids.allocate.command({
174 fixtureName: "delegated-completion",
175 commandName: "dispatch",
176 });
177 yield* threads
178 .dispatch({
179 type: "message.dispatch",
180 commandId,
181 threadId: request.threadId,
182 messageId: delivery.messageId,
183 text: delegatedCompletionText(delivery.taskIds),
184 attachments: [],
185 dispatchMode: { type: "queue_after_active" },
186 createdBy: "agent",
187 creationSource: "server",
188 delegatedCompletion: {
189 parentRunId: request.delegatedCompletion.parentRunId,
190 generation: delivery.generation,
191 taskIds: delivery.taskIds,
192 },
193 })
194 .pipe(
195 Effect.catchTags({
196 ThreadManagementProviderInstanceDisabledError: (error) =>
197 Effect.logInfo(
198 "orchestration-v2.provider-continuation.delegated-completion-disabled",
199 {
200 threadId: request.threadId,
201 projectId: error.projectId,
202 instanceId: error.instanceId,
203 },
204 ).pipe(
205 Effect.andThen(
206 Effect.forEach(
207 delivery.taskIds,
208 (taskId) =>
209 ids.allocate
210 .command({
211 fixtureName: "delegated-completion",
212 commandName: "dispose-disabled",
213 })
214 .pipe(
215 Effect.flatMap((disposeCommandId) =>
216 threads.dispatch({
217 type: "delegated_task.completion-delivery.dispose",
218 commandId: disposeCommandId,
219 parentThreadId: request.threadId,
220 taskId,
221 }),
222 ),
223 ),
224 { discard: true },
225 ),
226 ),
227 Effect.andThen(
228 request.clearIfCurrent !== undefined
229 ? request.clearIfCurrent()
230 : request.dispatchIfCurrent !== undefined
231 ? request.dispatchIfCurrent(Effect.void)
232 : Effect.void,
233 ),
234 ),
235 }),
236 );
237 yield* clearRetryAttempt(retryKey);
238 return;
239 }
240 const messageId =
241 request.messageId ??
242 (yield* ids.allocate.message({
243 threadId: request.threadId,
244 ordinal: projection.messages.length + 1,
245 }));
246 const commandId = CommandId.make(`provider-continuation:${messageId}`);
247 const dispatch = threads
248 .dispatch({
249 type: "message.dispatch",
250 commandId,
251 threadId: request.threadId,
252 messageId,
253 text: request.detail ?? CONTINUATION_MESSAGE_TEXT,
254 notification: request.notification ?? {
255 source: { kind: "background_task" },
256 outcome: "updated",
257 summary: "Background activity updated",
258 },
259 attachments: [],
260 dispatchMode: { type: "queue_after_active" },
261 createdBy: "agent",
262 // Only buffered wakes use creationSource "provider"; Claude would drain an empty buffer instead of prompting message_text.
263 creationSource: request.delivery === "message_text" ? "server" : "provider",
264 })
265 .pipe(Effect.tap((dispatched) => releaseIfNeverStarted(request, messageId, dispatched)));
266 // Catch after dispatchIfCurrent so adapters see raw failures first; clearIfCurrent is idempotent and covers others.
267 const logInstanceDisabled = (
268 error: ThreadManagementService.ThreadManagementProviderInstanceDisabledError,
269 ) =>
270 Effect.logInfo("orchestration-v2.provider-continuation.instance-disabled", {
271 threadId: request.threadId,
272 providerThreadId: request.providerThreadId,
273 projectId: error.projectId,
274 instanceId: error.instanceId,
275 }).pipe(
276 Effect.andThen(
277 request.clearIfCurrent !== undefined ? request.clearIfCurrent() : Effect.void,
278 ),
279 );
280 if (request.dispatchIfCurrent === undefined) {
281 yield* dispatch.pipe(
282 Effect.catchTags({
283 ThreadManagementProviderInstanceDisabledError: logInstanceDisabled,
284 }),
285 );
286 return;
287 }
288 yield* request.dispatchIfCurrent(dispatch).pipe(
289 Effect.catchTags({
290 ThreadManagementProviderInstanceDisabledError: logInstanceDisabled,
291 }),
292 );
293 },
294 );
295
296 yield* requests.take.pipe(
297 Effect.flatMap((request) =>
298 dispatchContinuation(request).pipe(
299 Effect.catchCause((cause) =>
300 Effect.gen(function* () {
301 yield* Effect.logWarning("orchestration-v2.provider-continuation.dispatch-failed", {
302 threadId: request.threadId,
303 providerThreadId: request.providerThreadId,
304 cause,
305 });
306 if (request.delegatedCompletion !== undefined) {
307 const completion = request.delegatedCompletion;
308 const retryKey = delegatedCompletionRetryKey(request, completion);
309 const retryDelay = yield* nextRetryDelay(retryKey);
310 yield* Effect.gen(function* () {
311 yield* Effect.sleep(`${retryDelay} millis`);
312 const projection = yield* threads.getThreadRecords(
313 request.threadId,
314 ["messages", "runs", "providerTurns"],
315 {
316 messageIds:
317 request.delegatedCompletion === undefined
318 ? []
319 : [request.delegatedCompletion.messageId],
320 },
321 );
322 if (currentDelegatedCompletionDelivery(projection, completion) !== undefined) {
323 yield* requests.offer(request);
324 } else {
325 yield* clearRetryAttempt(retryKey);
326 }
327 }).pipe(
328 Effect.catchCause((retryCause) =>
329 Effect.logWarning("orchestration-v2.provider-continuation.retry-check-failed", {
330 threadId: request.threadId,
331 providerThreadId: request.providerThreadId,
332 cause: retryCause,
333 }).pipe(Effect.andThen(requests.offer(request))),
334 ),
335 Effect.forkScoped,
336 );
337 } else if (request.clearIfCurrent !== undefined) {
338 yield* request.clearIfCurrent();
339 }
340 }),
341 ),
342 // A dispatch can wait on its parent's lock, so only its own thread's later requests queue behind it.
343 (handled) => continuations.run(request.threadId, handled),
344 ),
345 ),
346 Effect.forever,
347 Effect.forkScoped,
348 );
349 }),
350 );