The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import { |
| 2 | CommandId, |
| 3 | type MessageId, |
| 4 | type OrchestrationV2ThreadProjection, |
| 5 | type ThreadId, |
| 6 | } from "@t3tools/contracts"; |
| 7 | import * as Effect from "effect/Effect"; |
| 8 | import * as Layer from "effect/Layer"; |
| 9 | import * as Option from "effect/Option"; |
| 10 | import * as Ref from "effect/Ref"; |
| 11 | import * as Stream from "effect/Stream"; |
| 12 | |
| 13 | import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; |
| 14 | import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests"; |
| 15 | import * as KeyedLock from "@t3tools/shared/KeyedLock"; |
| 16 | import type * as Orchestrator from "./Orchestrator.ts"; |
| 17 | import * as ThreadManagementService from "./ThreadManagementService.ts"; |
| 18 | import { isUndeliveredMailboxSteer } from "./NotificationMailbox.ts"; |
| 19 | |
| 20 | const CONTINUATION_MESSAGE_TEXT = "Background task completed."; |
| 21 | |
| 22 | function delegatedCompletionText(taskIds: ReadonlyArray<string>): string { |
| 23 | const taskList = taskIds.join(", "); |
| 24 | return taskIds.length === 1 |
| 25 | ? `Delegated task ${taskList} reached a terminal state. Use task_status with taskId ${taskList} to read the result.` |
| 26 | : `Delegated tasks ${taskList} reached terminal states. Use task_status with each taskId to read the results.`; |
| 27 | } |
| 28 | |
| 29 | function currentDelegatedCompletionDelivery( |
| 30 | projection: Pick<OrchestrationV2ThreadProjection, "messages" | "runs" | "providerTurns">, |
| 31 | completion: NonNullable< |
| 32 | ProviderContinuationRequests.ProviderContinuationRequest["delegatedCompletion"] |
| 33 | >, |
| 34 | ) { |
| 35 | const sourceRun = projection.runs.find((candidate) => candidate.id === completion.parentRunId); |
| 36 | const delivery = sourceRun?.delegatedCompletion?.delivery; |
| 37 | const alreadyDispatched = projection.messages.some( |
| 38 | (message) => message.id === completion.messageId, |
| 39 | ); |
| 40 | if ( |
| 41 | sourceRun?.delegatedCompletion?.disposition !== "open" || |
| 42 | delivery === null || |
| 43 | delivery === undefined || |
| 44 | delivery.generation !== completion.generation || |
| 45 | delivery.messageId !== completion.messageId || |
| 46 | (alreadyDispatched && !isUndeliveredMailboxSteer(projection, completion.messageId)) |
| 47 | ) { |
| 48 | return undefined; |
| 49 | } |
| 50 | return delivery; |
| 51 | } |
| 52 | |
| 53 | function delegatedCompletionRetryKey( |
| 54 | request: ProviderContinuationRequests.ProviderContinuationRequest, |
| 55 | completion: NonNullable< |
| 56 | ProviderContinuationRequests.ProviderContinuationRequest["delegatedCompletion"] |
| 57 | >, |
| 58 | ): string { |
| 59 | return `${request.threadId}:${completion.parentRunId}:${completion.generation}:${completion.messageId}`; |
| 60 | } |
| 61 | |
| 62 | export const layer = Layer.effectDiscard( |
| 63 | Effect.gen(function* () { |
| 64 | const ids = yield* IdAllocator.IdAllocatorV2; |
| 65 | const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests; |
| 66 | const threads = yield* ThreadManagementService.ThreadManagementService; |
| 67 | const retryAttempts = yield* Ref.make(new Map<string, number>()); |
| 68 | const continuations = yield* KeyedLock.makeRunner<ThreadId>(); |
| 69 | |
| 70 | const clearRetryAttempt = (key: string) => |
| 71 | Ref.update(retryAttempts, (current) => { |
| 72 | if (!current.has(key)) return current; |
| 73 | const updated = new Map(current); |
| 74 | updated.delete(key); |
| 75 | return updated; |
| 76 | }); |
| 77 | |
| 78 | const nextRetryDelay = (key: string) => |
| 79 | Ref.modify(retryAttempts, (current) => { |
| 80 | const attempt = current.get(key) ?? 0; |
| 81 | const updated = new Map(current); |
| 82 | updated.set(key, attempt + 1); |
| 83 | return [Math.min(100 * 2 ** Math.min(attempt, 6), 5_000), updated] as const; |
| 84 | }); |
| 85 | |
| 86 | // Cancelling or stopping a queued continuation ends it unstarted; only the request can release the adapter's wake latch. |
| 87 | const releaseIfNeverStarted = Effect.fn("ProviderContinuationService.releaseIfNeverStarted")( |
| 88 | function* ( |
| 89 | request: ProviderContinuationRequests.ProviderContinuationRequest, |
| 90 | messageId: MessageId, |
| 91 | dispatched: Orchestrator.OrchestratorV2DispatchResult, |
| 92 | ) { |
| 93 | const onRunEndedWithoutStarting = request.onRunEndedWithoutStarting; |
| 94 | if (onRunEndedWithoutStarting === undefined) return; |
| 95 | const runId = dispatched.storedEvents.flatMap(({ event }) => |
| 96 | event.type === "run.created" && event.payload.userMessageId === messageId |
| 97 | ? [event.payload.id] |
| 98 | : [], |
| 99 | )[0]; |
| 100 | if (runId === undefined) return; |
| 101 | yield* threads |
| 102 | .streamStoredEventsFrom({ |
| 103 | threadId: request.threadId, |
| 104 | afterSequence: dispatched.sequence, |
| 105 | eventType: "run.updated", |
| 106 | }) |
| 107 | .pipe( |
| 108 | Stream.filter( |
| 109 | ({ event }) => |
| 110 | event.type === "run.updated" && |
| 111 | event.payload.id === runId && |
| 112 | ThreadManagementService.isTerminalRunStatus(event.payload.status), |
| 113 | ), |
| 114 | Stream.runHead, |
| 115 | Effect.flatMap((ended) => |
| 116 | Option.isSome(ended) && |
| 117 | ended.value.event.type === "run.updated" && |
| 118 | ended.value.event.payload.startedAt === null |
| 119 | ? onRunEndedWithoutStarting() |
| 120 | : Effect.void, |
| 121 | ), |
| 122 | Effect.catchCause((cause) => |
| 123 | Effect.logWarning("orchestration-v2.provider-continuation.run-watch-failed", { |
| 124 | threadId: request.threadId, |
| 125 | providerThreadId: request.providerThreadId, |
| 126 | cause, |
| 127 | }), |
| 128 | ), |
| 129 | Effect.forkScoped, |
| 130 | ); |
| 131 | }, |
| 132 | ); |
| 133 | |
| 134 | const dispatchContinuation = Effect.fn("ProviderContinuationService.dispatchContinuation")( |
| 135 | function* (request: ProviderContinuationRequests.ProviderContinuationRequest) { |
| 136 | const projection = yield* threads.getThreadRecords( |
| 137 | request.threadId, |
| 138 | ["messages", "runs", "providerTurns"], |
| 139 | { |
| 140 | messageIds: |
| 141 | request.delegatedCompletion === undefined |
| 142 | ? [] |
| 143 | : [request.delegatedCompletion.messageId], |
| 144 | }, |
| 145 | ); |
| 146 | if (projection.thread.archivedAt !== null || projection.thread.deletedAt !== null) { |
| 147 | yield* Effect.logInfo("orchestration-v2.provider-continuation.thread-archived", { |
| 148 | threadId: request.threadId, |
| 149 | providerThreadId: request.providerThreadId, |
| 150 | }); |
| 151 | if (request.delegatedCompletion !== undefined) { |
| 152 | yield* clearRetryAttempt( |
| 153 | delegatedCompletionRetryKey(request, request.delegatedCompletion), |
| 154 | ); |
| 155 | } |
| 156 | if (request.clearIfCurrent !== undefined) { |
| 157 | yield* request.clearIfCurrent(); |
| 158 | } else if (request.dispatchIfCurrent !== undefined) { |
| 159 | yield* request.dispatchIfCurrent(Effect.void); |
| 160 | } |
| 161 | return; |
| 162 | } |
| 163 | if (request.delegatedCompletion !== undefined) { |
| 164 | const retryKey = delegatedCompletionRetryKey(request, request.delegatedCompletion); |
| 165 | const delivery = currentDelegatedCompletionDelivery( |
| 166 | projection, |
| 167 | request.delegatedCompletion, |
| 168 | ); |
| 169 | if (delivery === undefined) { |
| 170 | yield* clearRetryAttempt(retryKey); |
| 171 | return; |
| 172 | } |
| 173 | const commandId = yield* ids.allocate.command({ |
| 174 | fixtureName: "delegated-completion", |
| 175 | commandName: "dispatch", |
| 176 | }); |
| 177 | yield* threads |
| 178 | .dispatch({ |
| 179 | type: "message.dispatch", |
| 180 | commandId, |
| 181 | threadId: request.threadId, |
| 182 | messageId: delivery.messageId, |
| 183 | text: delegatedCompletionText(delivery.taskIds), |
| 184 | attachments: [], |
| 185 | dispatchMode: { type: "queue_after_active" }, |
| 186 | createdBy: "agent", |
| 187 | creationSource: "server", |
| 188 | delegatedCompletion: { |
| 189 | parentRunId: request.delegatedCompletion.parentRunId, |
| 190 | generation: delivery.generation, |
| 191 | taskIds: delivery.taskIds, |
| 192 | }, |
| 193 | }) |
| 194 | .pipe( |
| 195 | Effect.catchTags({ |
| 196 | ThreadManagementProviderInstanceDisabledError: (error) => |
| 197 | Effect.logInfo( |
| 198 | "orchestration-v2.provider-continuation.delegated-completion-disabled", |
| 199 | { |
| 200 | threadId: request.threadId, |
| 201 | projectId: error.projectId, |
| 202 | instanceId: error.instanceId, |
| 203 | }, |
| 204 | ).pipe( |
| 205 | Effect.andThen( |
| 206 | Effect.forEach( |
| 207 | delivery.taskIds, |
| 208 | (taskId) => |
| 209 | ids.allocate |
| 210 | .command({ |
| 211 | fixtureName: "delegated-completion", |
| 212 | commandName: "dispose-disabled", |
| 213 | }) |
| 214 | .pipe( |
| 215 | Effect.flatMap((disposeCommandId) => |
| 216 | threads.dispatch({ |
| 217 | type: "delegated_task.completion-delivery.dispose", |
| 218 | commandId: disposeCommandId, |
| 219 | parentThreadId: request.threadId, |
| 220 | taskId, |
| 221 | }), |
| 222 | ), |
| 223 | ), |
| 224 | { discard: true }, |
| 225 | ), |
| 226 | ), |
| 227 | Effect.andThen( |
| 228 | request.clearIfCurrent !== undefined |
| 229 | ? request.clearIfCurrent() |
| 230 | : request.dispatchIfCurrent !== undefined |
| 231 | ? request.dispatchIfCurrent(Effect.void) |
| 232 | : Effect.void, |
| 233 | ), |
| 234 | ), |
| 235 | }), |
| 236 | ); |
| 237 | yield* clearRetryAttempt(retryKey); |
| 238 | return; |
| 239 | } |
| 240 | const messageId = |
| 241 | request.messageId ?? |
| 242 | (yield* ids.allocate.message({ |
| 243 | threadId: request.threadId, |
| 244 | ordinal: projection.messages.length + 1, |
| 245 | })); |
| 246 | const commandId = CommandId.make(`provider-continuation:${messageId}`); |
| 247 | const dispatch = threads |
| 248 | .dispatch({ |
| 249 | type: "message.dispatch", |
| 250 | commandId, |
| 251 | threadId: request.threadId, |
| 252 | messageId, |
| 253 | text: request.detail ?? CONTINUATION_MESSAGE_TEXT, |
| 254 | notification: request.notification ?? { |
| 255 | source: { kind: "background_task" }, |
| 256 | outcome: "updated", |
| 257 | summary: "Background activity updated", |
| 258 | }, |
| 259 | attachments: [], |
| 260 | dispatchMode: { type: "queue_after_active" }, |
| 261 | createdBy: "agent", |
| 262 | // Only buffered wakes use creationSource "provider"; Claude would drain an empty buffer instead of prompting message_text. |
| 263 | creationSource: request.delivery === "message_text" ? "server" : "provider", |
| 264 | }) |
| 265 | .pipe(Effect.tap((dispatched) => releaseIfNeverStarted(request, messageId, dispatched))); |
| 266 | // Catch after dispatchIfCurrent so adapters see raw failures first; clearIfCurrent is idempotent and covers others. |
| 267 | const logInstanceDisabled = ( |
| 268 | error: ThreadManagementService.ThreadManagementProviderInstanceDisabledError, |
| 269 | ) => |
| 270 | Effect.logInfo("orchestration-v2.provider-continuation.instance-disabled", { |
| 271 | threadId: request.threadId, |
| 272 | providerThreadId: request.providerThreadId, |
| 273 | projectId: error.projectId, |
| 274 | instanceId: error.instanceId, |
| 275 | }).pipe( |
| 276 | Effect.andThen( |
| 277 | request.clearIfCurrent !== undefined ? request.clearIfCurrent() : Effect.void, |
| 278 | ), |
| 279 | ); |
| 280 | if (request.dispatchIfCurrent === undefined) { |
| 281 | yield* dispatch.pipe( |
| 282 | Effect.catchTags({ |
| 283 | ThreadManagementProviderInstanceDisabledError: logInstanceDisabled, |
| 284 | }), |
| 285 | ); |
| 286 | return; |
| 287 | } |
| 288 | yield* request.dispatchIfCurrent(dispatch).pipe( |
| 289 | Effect.catchTags({ |
| 290 | ThreadManagementProviderInstanceDisabledError: logInstanceDisabled, |
| 291 | }), |
| 292 | ); |
| 293 | }, |
| 294 | ); |
| 295 | |
| 296 | yield* requests.take.pipe( |
| 297 | Effect.flatMap((request) => |
| 298 | dispatchContinuation(request).pipe( |
| 299 | Effect.catchCause((cause) => |
| 300 | Effect.gen(function* () { |
| 301 | yield* Effect.logWarning("orchestration-v2.provider-continuation.dispatch-failed", { |
| 302 | threadId: request.threadId, |
| 303 | providerThreadId: request.providerThreadId, |
| 304 | cause, |
| 305 | }); |
| 306 | if (request.delegatedCompletion !== undefined) { |
| 307 | const completion = request.delegatedCompletion; |
| 308 | const retryKey = delegatedCompletionRetryKey(request, completion); |
| 309 | const retryDelay = yield* nextRetryDelay(retryKey); |
| 310 | yield* Effect.gen(function* () { |
| 311 | yield* Effect.sleep(`${retryDelay} millis`); |
| 312 | const projection = yield* threads.getThreadRecords( |
| 313 | request.threadId, |
| 314 | ["messages", "runs", "providerTurns"], |
| 315 | { |
| 316 | messageIds: |
| 317 | request.delegatedCompletion === undefined |
| 318 | ? [] |
| 319 | : [request.delegatedCompletion.messageId], |
| 320 | }, |
| 321 | ); |
| 322 | if (currentDelegatedCompletionDelivery(projection, completion) !== undefined) { |
| 323 | yield* requests.offer(request); |
| 324 | } else { |
| 325 | yield* clearRetryAttempt(retryKey); |
| 326 | } |
| 327 | }).pipe( |
| 328 | Effect.catchCause((retryCause) => |
| 329 | Effect.logWarning("orchestration-v2.provider-continuation.retry-check-failed", { |
| 330 | threadId: request.threadId, |
| 331 | providerThreadId: request.providerThreadId, |
| 332 | cause: retryCause, |
| 333 | }).pipe(Effect.andThen(requests.offer(request))), |
| 334 | ), |
| 335 | Effect.forkScoped, |
| 336 | ); |
| 337 | } else if (request.clearIfCurrent !== undefined) { |
| 338 | yield* request.clearIfCurrent(); |
| 339 | } |
| 340 | }), |
| 341 | ), |
| 342 | // A dispatch can wait on its parent's lock, so only its own thread's later requests queue behind it. |
| 343 | (handled) => continuations.run(request.threadId, handled), |
| 344 | ), |
| 345 | ), |
| 346 | Effect.forever, |
| 347 | Effect.forkScoped, |
| 348 | ); |
| 349 | }), |
| 350 | ); |