The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 03:37 UTC
| 1 | import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError"; |
| 2 | import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; |
| 3 | import { |
| 4 | CommandId, |
| 5 | isOrchestrationV2WorkActive, |
| 6 | latestProviderTurnForAttempt, |
| 7 | MessageId, |
| 8 | type OrchestrationV2Run, |
| 9 | type RunId, |
| 10 | type ThreadId, |
| 11 | } from "@t3tools/contracts"; |
| 12 | import * as DateTime from "effect/DateTime"; |
| 13 | import * as Effect from "effect/Effect"; |
| 14 | import type { |
| 15 | ProjectionRecordField, |
| 16 | ProjectionRecordFilter, |
| 17 | ProjectionRecords, |
| 18 | ProjectionRuntimeRecoveryState, |
| 19 | } from "./ProjectionStore.ts"; |
| 20 | |
| 21 | import * as ServerSettings from "../serverSettings.ts"; |
| 22 | import { isNativeMaintenanceCommand } from "./CommandPolicy.ts"; |
| 23 | import * as ThreadManagementService from "./ThreadManagementService.ts"; |
| 24 | import { |
| 25 | awaitingProviderThreadId, |
| 26 | cancelledRosterTaskWork, |
| 27 | cancelledTurnItemWork, |
| 28 | continuationFailedBeforePrompt, |
| 29 | isAwaitedWork, |
| 30 | isRestartContinuationSource, |
| 31 | cancelledBeforeStart, |
| 32 | isRestartNoteSource, |
| 33 | neverStarted, |
| 34 | restartCancelledBackgroundWorkNote, |
| 35 | restartContinuationNote, |
| 36 | } from "./RestartBackgroundNote.ts"; |
| 37 | |
| 38 | const CONTINUE_PROMPT = |
| 39 | "Continue where you left off. Your last turn was cut short, so a command or other action it started may have finished or may still be running. Before you repeat one that changes state, confirm it did not take effect and is not still running; if you cannot tell, say so instead of repeating it."; |
| 40 | |
| 41 | const MAX_CONTINUATION_CHAIN = 3; |
| 42 | const CONTINUATION_LOOP_WINDOW_MS = 5 * 60 * 1_000; |
| 43 | |
| 44 | export function restartContinuationBlocked( |
| 45 | projection: Pick< |
| 46 | ProjectionRuntimeRecoveryState, |
| 47 | "thread" | "runtimeRequests" | "providerTurns" | "turnItems" |
| 48 | >, |
| 49 | source: OrchestrationV2Run, |
| 50 | now: DateTime.Utc, |
| 51 | settled: boolean, |
| 52 | ): boolean { |
| 53 | const { thread } = projection; |
| 54 | if ( |
| 55 | thread.archivedAt !== null || |
| 56 | thread.deletedAt !== null || |
| 57 | thread.settledOverride === "settled" || |
| 58 | (thread.snoozedUntil != null && DateTime.isGreaterThan(thread.snoozedUntil, now)) |
| 59 | ) |
| 60 | return true; |
| 61 | const turn = latestProviderTurnForAttempt(projection.providerTurns, source.activeAttemptId); |
| 62 | if ( |
| 63 | !settled && |
| 64 | turn !== undefined && |
| 65 | (turn.status === "completed" || turn.status === "failed" || turn.status === "interrupted") |
| 66 | ) |
| 67 | return true; |
| 68 | const waitingRequest = projection.runtimeRequests.some( |
| 69 | (request) => |
| 70 | request.kind !== "dynamic_tool_call" && |
| 71 | request.kind !== "auth_refresh" && |
| 72 | request.responseCapability.type !== "message" && |
| 73 | (request.status === "pending" || |
| 74 | ((request.status === "expired" || request.status === "cancelled") && |
| 75 | request.responseCapability.type === "not_resumable" && |
| 76 | turn !== undefined && |
| 77 | turn.id === request.providerTurnId)), |
| 78 | ); |
| 79 | if (waitingRequest) return true; |
| 80 | return projection.turnItems.some( |
| 81 | (item) => |
| 82 | item.type === "secret_request" && |
| 83 | item.runId === source.id && |
| 84 | (item.secretStatus === "pending" || |
| 85 | (item.secretStatus === "cancelled" && |
| 86 | item.completedAt !== null && |
| 87 | source.completedAt !== null && |
| 88 | DateTime.Equivalence(item.completedAt, source.completedAt))), |
| 89 | ); |
| 90 | } |
| 91 | |
| 92 | /** A Stop from the user, or from the parent that delegated the task, that a restart cut short. */ |
| 93 | export const restartStopRequested = Effect.fn("RestartContinuation.restartStopRequested")( |
| 94 | function* <E>( |
| 95 | getThreadRecords: <K extends ProjectionRecordField>( |
| 96 | threadId: ThreadId, |
| 97 | fields: ReadonlyArray<K>, |
| 98 | filter: ProjectionRecordFilter, |
| 99 | ) => Effect.Effect<ProjectionRecords<K>, E>, |
| 100 | thread: Pick<ProjectionRuntimeRecoveryState["thread"], "id" | "lineage" | "forkedFrom">, |
| 101 | source: Pick<OrchestrationV2Run, "id" | "delegatedTaskId">, |
| 102 | ) { |
| 103 | const { turnItems } = yield* getThreadRecords(thread.id, ["turnItems"], { |
| 104 | turnItemRunIds: [source.id], |
| 105 | turnItemTypes: ["run_interrupt_request"], |
| 106 | }); |
| 107 | if (turnItems.some((item) => item.runId === source.id && item.type === "run_interrupt_request")) |
| 108 | return true; |
| 109 | const { lineage, forkedFrom } = thread; |
| 110 | const taskId = |
| 111 | source.delegatedTaskId ?? (forkedFrom?.type === "node" ? forkedFrom.nodeId : undefined); |
| 112 | if ( |
| 113 | lineage.relationshipToParent !== "subagent" || |
| 114 | lineage.parentThreadId === null || |
| 115 | taskId === undefined |
| 116 | ) |
| 117 | return false; |
| 118 | const task = (yield* getThreadRecords(lineage.parentThreadId, ["subagents"], { |
| 119 | subagentIds: [taskId], |
| 120 | })).subagents.find((candidate) => candidate.id === taskId); |
| 121 | // A settled task no longer owns later work in its thread. |
| 122 | if ( |
| 123 | task === undefined || |
| 124 | task.runId === null || |
| 125 | task.result !== null || |
| 126 | !isOrchestrationV2WorkActive(task.status) |
| 127 | ) |
| 128 | return false; |
| 129 | return ( |
| 130 | (yield* getThreadRecords(lineage.parentThreadId, ["turnItems"], { |
| 131 | turnItemTypes: ["run_interrupt_request"], |
| 132 | turnItemRunIds: [task.runId], |
| 133 | })).turnItems.length > 0 |
| 134 | ); |
| 135 | }, |
| 136 | ); |
| 137 | |
| 138 | /** App-owned delegations continue in their own threads and report back. */ |
| 139 | function hasOpenAwaitedWork( |
| 140 | projection: Pick<ProjectionRuntimeRecoveryState, "runs" | "turnItems">, |
| 141 | providerThread: ProjectionRuntimeRecoveryState["providerThreads"][number], |
| 142 | ): boolean { |
| 143 | const openItem = projection.turnItems.some((item) => { |
| 144 | if (item.status !== "pending" && item.status !== "running" && item.status !== "waiting") |
| 145 | return false; |
| 146 | if (item.type === "subagent" && item.origin === "app_owned" && item.childThreadId !== null) |
| 147 | return false; |
| 148 | const work = cancelledTurnItemWork(item); |
| 149 | return ( |
| 150 | work !== undefined && |
| 151 | isAwaitedWork(work) && |
| 152 | awaitingProviderThreadId(item, projection.runs) === providerThread.id |
| 153 | ); |
| 154 | }); |
| 155 | return ( |
| 156 | openItem || |
| 157 | (providerThread.ownerNodeId === null && |
| 158 | (providerThread.pendingBackgroundTasks ?? []).some((task) => |
| 159 | isAwaitedWork(cancelledRosterTaskWork(task)), |
| 160 | )) |
| 161 | ); |
| 162 | } |
| 163 | |
| 164 | /** Settled runs require a live session so stale crash records cannot wake them. */ |
| 165 | export function restartContinuationRun( |
| 166 | projection: Pick< |
| 167 | ProjectionRuntimeRecoveryState, |
| 168 | | "thread" |
| 169 | | "runs" |
| 170 | | "providerThreads" |
| 171 | | "providerSessions" |
| 172 | | "providerTurns" |
| 173 | | "runtimeRequests" |
| 174 | | "turnItems" |
| 175 | >, |
| 176 | now: DateTime.Utc, |
| 177 | ): OrchestrationV2Run | undefined { |
| 178 | const run = projection.runs.reduce<OrchestrationV2Run | undefined>( |
| 179 | (latest, candidate) => |
| 180 | candidate.status !== "queued" && |
| 181 | !neverStarted(candidate) && |
| 182 | (!latest || runRanAfter(candidate, latest)) |
| 183 | ? candidate |
| 184 | : latest, |
| 185 | undefined, |
| 186 | ); |
| 187 | if (!run) return; |
| 188 | const preparedContinuation = |
| 189 | run.status === "starting" && run.restartContinuationOfRunId !== undefined; |
| 190 | const settled = run.status === "completed" || run.status === "waiting"; |
| 191 | if (run.status !== "running" && !preparedContinuation && !settled) return; |
| 192 | if (restartContinuationBlocked(projection, run, now, settled)) return; |
| 193 | const liveTurnRequired = !preparedContinuation && !settled; |
| 194 | if (projection.thread.providerInstanceId !== run.providerInstanceId) return; |
| 195 | const providerThread = projection.providerThreads.find( |
| 196 | (thread) => thread.id === run.providerThreadId, |
| 197 | ); |
| 198 | if ( |
| 199 | !providerThread || |
| 200 | providerThread.appThreadId !== projection.thread.id || |
| 201 | providerThread.ownerNodeId !== null || |
| 202 | providerThread.providerInstanceId !== run.providerInstanceId || |
| 203 | providerThread.nativeThreadRef?.nativeId == null || |
| 204 | providerThread.nativeThreadRef.strength !== "strong" || |
| 205 | providerThread.nativeThreadRef.driver !== providerThread.driver || |
| 206 | (liveTurnRequired && providerThread.status !== "active") || |
| 207 | providerThread.status === "closed" || |
| 208 | providerThread.status === "archived" |
| 209 | ) |
| 210 | return; |
| 211 | const session = projection.providerSessions.find( |
| 212 | (candidate) => candidate.id === providerThread.providerSessionId, |
| 213 | ); |
| 214 | // Adapters remain ready during turns; only stopped or failed sessions rule out liveness. |
| 215 | if ( |
| 216 | session === undefined || |
| 217 | session.providerInstanceId !== run.providerInstanceId || |
| 218 | session.driver !== providerThread.driver || |
| 219 | (!preparedContinuation && (session.status === "stopped" || session.status === "error")) |
| 220 | ) |
| 221 | return; |
| 222 | if (settled) return hasOpenAwaitedWork(projection, providerThread) ? run : undefined; |
| 223 | if ( |
| 224 | liveTurnRequired && |
| 225 | !projection.providerTurns.some( |
| 226 | (turn) => |
| 227 | turn.providerThreadId === providerThread.id && |
| 228 | turn.runAttemptId === run.activeAttemptId && |
| 229 | turn.status === "running", |
| 230 | ) |
| 231 | ) |
| 232 | return; |
| 233 | return run; |
| 234 | } |
| 235 | |
| 236 | export const continueRestartedRun = Effect.fn("RestartContinuation.continueRestartedRun")( |
| 237 | function* (input: { |
| 238 | readonly threadId: ThreadId; |
| 239 | readonly sourceRunId: RunId; |
| 240 | readonly processLossCount?: number; |
| 241 | }) { |
| 242 | const settings = yield* ServerSettings.ServerSettingsService; |
| 243 | const enabled = yield* settings.getSettings.pipe(Effect.orElseSucceed(() => null)); |
| 244 | if (!enabled) return; |
| 245 | const threads = yield* ThreadManagementService.ThreadManagementService; |
| 246 | const messageId = MessageId.make(`message:restart-continuation:${input.sourceRunId}`); |
| 247 | const projection = yield* threads.getThreadRecords( |
| 248 | input.threadId, |
| 249 | ["messages", "runs", "providerTurns", "attempts", "runtimeRequests", "turnItems"], |
| 250 | { messageIds: [messageId], turnItemTypes: ["secret_request"] }, |
| 251 | ); |
| 252 | if ( |
| 253 | !resolveProjectSettings(enabled, projection.thread.projectId).settings |
| 254 | .continueThreadsAfterServerUpdate |
| 255 | ) |
| 256 | return; |
| 257 | if (projection.messages.some((message) => message.id === messageId)) return; |
| 258 | const source = projection.runs.find((run) => run.id === input.sourceRunId); |
| 259 | if ( |
| 260 | !source || |
| 261 | !( |
| 262 | isRestartContinuationSource(source, projection.providerTurns) || |
| 263 | continuationFailedBeforePrompt(source, projection.providerTurns) |
| 264 | ) || |
| 265 | restartContinuationBlocked( |
| 266 | projection, |
| 267 | source, |
| 268 | yield* DateTime.now, |
| 269 | isRestartNoteSource(source, projection.providerTurns), |
| 270 | ) |
| 271 | ) |
| 272 | return; |
| 273 | if ( |
| 274 | projection.runs.some( |
| 275 | (run) => |
| 276 | run.id !== source.id && |
| 277 | run.status !== "queued" && |
| 278 | !cancelledBeforeStart(run) && |
| 279 | runRanAfter(run, source), |
| 280 | ) |
| 281 | ) |
| 282 | return; |
| 283 | if (projection.thread.providerInstanceId !== source.providerInstanceId) return; |
| 284 | // A turn that keeps crashing the server would otherwise restart it forever. |
| 285 | let chainLength = 0; |
| 286 | const loopWindowStart = |
| 287 | DateTime.toEpochMillis(source.completedAt ?? source.requestedAt) - |
| 288 | CONTINUATION_LOOP_WINDOW_MS; |
| 289 | for ( |
| 290 | let run: OrchestrationV2Run | undefined = source; |
| 291 | run?.restartContinuationOfRunId !== undefined && chainLength < MAX_CONTINUATION_CHAIN; |
| 292 | run = projection.runs.find((candidate) => candidate.id === run?.restartContinuationOfRunId) |
| 293 | ) { |
| 294 | if (DateTime.toEpochMillis(run.requestedAt) < loopWindowStart) break; |
| 295 | chainLength += 1; |
| 296 | } |
| 297 | if ( |
| 298 | input.processLossCount === undefined |
| 299 | ? chainLength >= MAX_CONTINUATION_CHAIN |
| 300 | : input.processLossCount > MAX_CONTINUATION_CHAIN |
| 301 | ) { |
| 302 | yield* Effect.logWarning("Not continuing a run restarted too many times in a row", { |
| 303 | threadId: input.threadId, |
| 304 | sourceRunId: input.sourceRunId, |
| 305 | }); |
| 306 | return; |
| 307 | } |
| 308 | if (yield* restartStopRequested(threads.getThreadRecords, projection.thread, source)) return; |
| 309 | const sourceMessage = (yield* threads.getThreadRecords(input.threadId, ["messages"], { |
| 310 | messageIds: [source.userMessageId], |
| 311 | })).messages.find((message) => message.id === source.userMessageId); |
| 312 | if (sourceMessage !== undefined && isNativeMaintenanceCommand(sourceMessage)) return; |
| 313 | const note = restartContinuationNote( |
| 314 | source, |
| 315 | projection.runs, |
| 316 | projection.providerTurns, |
| 317 | projection.attempts, |
| 318 | ); |
| 319 | const noteText = |
| 320 | note.work.length === 0 ? undefined : restartCancelledBackgroundWorkNote(note.work); |
| 321 | const text = |
| 322 | noteText === undefined |
| 323 | ? CONTINUE_PROMPT |
| 324 | : note.settled |
| 325 | ? noteText |
| 326 | : `${noteText}\n\n${CONTINUE_PROMPT}`; |
| 327 | yield* threads |
| 328 | .dispatch({ |
| 329 | type: "message.dispatch", |
| 330 | commandId: CommandId.make(`command:restart-continuation:${input.sourceRunId}`), |
| 331 | threadId: input.threadId, |
| 332 | messageId, |
| 333 | text, |
| 334 | notification: { |
| 335 | source: { kind: "system" }, |
| 336 | outcome: "updated", |
| 337 | summary: note.settled |
| 338 | ? "T3 Code restarted and stopped background work" |
| 339 | : "T3 Code restarted and resumed this turn", |
| 340 | detail: text, |
| 341 | }, |
| 342 | attachments: [], |
| 343 | modelSelection: source.modelSelection, |
| 344 | dispatchMode: { type: "start_immediately" }, |
| 345 | createdBy: "agent", |
| 346 | creationSource: "server", |
| 347 | restartContinuationOfRunId: input.sourceRunId, |
| 348 | }) |
| 349 | .pipe( |
| 350 | // Disabled instances, and threads above a policy ceiling, cannot resume; still run success |
| 351 | // cleanup so delegated children recover without futile retries. |
| 352 | Effect.catchTags({ |
| 353 | ThreadManagementProviderInstanceDisabledError: (error) => |
| 354 | Effect.logInfo("orchestration-v2.restart-continuation.instance-disabled", { |
| 355 | threadId: input.threadId, |
| 356 | sourceRunId: input.sourceRunId, |
| 357 | projectId: error.projectId, |
| 358 | instanceId: error.instanceId, |
| 359 | }), |
| 360 | OrchestratorRuntimeModeAboveCeilingError: () => |
| 361 | Effect.logInfo("orchestration-v2.restart-continuation.above-ceiling", { |
| 362 | threadId: input.threadId, |
| 363 | sourceRunId: input.sourceRunId, |
| 364 | }), |
| 365 | }), |
| 366 | ); |
| 367 | }, |
| 368 | // Declined child continuations still owe parent results; existing continuation runs settle them instead. |
| 369 | (effect, input) => |
| 370 | effect.pipe( |
| 371 | Effect.andThen( |
| 372 | Effect.gen(function* () { |
| 373 | const threads = yield* ThreadManagementService.ThreadManagementService; |
| 374 | yield* threads.recoverDelegatedTask(input.threadId, { sourceRunId: input.sourceRunId }); |
| 375 | }), |
| 376 | ), |
| 377 | ), |
| 378 | ); |