@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

main
ts 378 lines 13.9 KB
Raw
1 import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
2 import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
3 import {
4 CommandId,
5 isOrchestrationV2WorkActive,
6 latestProviderTurnForAttempt,
7 MessageId,
8 type OrchestrationV2Run,
9 type RunId,
10 type ThreadId,
11 } from "@t3tools/contracts";
12 import * as DateTime from "effect/DateTime";
13 import * as Effect from "effect/Effect";
14 import type {
15 ProjectionRecordField,
16 ProjectionRecordFilter,
17 ProjectionRecords,
18 ProjectionRuntimeRecoveryState,
19 } from "./ProjectionStore.ts";
20
21 import * as ServerSettings from "../serverSettings.ts";
22 import { isNativeMaintenanceCommand } from "./CommandPolicy.ts";
23 import * as ThreadManagementService from "./ThreadManagementService.ts";
24 import {
25 awaitingProviderThreadId,
26 cancelledRosterTaskWork,
27 cancelledTurnItemWork,
28 continuationFailedBeforePrompt,
29 isAwaitedWork,
30 isRestartContinuationSource,
31 cancelledBeforeStart,
32 isRestartNoteSource,
33 neverStarted,
34 restartCancelledBackgroundWorkNote,
35 restartContinuationNote,
36 } from "./RestartBackgroundNote.ts";
37
38 const CONTINUE_PROMPT =
39 "Continue where you left off. Your last turn was cut short, so a command or other action it started may have finished or may still be running. Before you repeat one that changes state, confirm it did not take effect and is not still running; if you cannot tell, say so instead of repeating it.";
40
41 const MAX_CONTINUATION_CHAIN = 3;
42 const CONTINUATION_LOOP_WINDOW_MS = 5 * 60 * 1_000;
43
44 export function restartContinuationBlocked(
45 projection: Pick<
46 ProjectionRuntimeRecoveryState,
47 "thread" | "runtimeRequests" | "providerTurns" | "turnItems"
48 >,
49 source: OrchestrationV2Run,
50 now: DateTime.Utc,
51 settled: boolean,
52 ): boolean {
53 const { thread } = projection;
54 if (
55 thread.archivedAt !== null ||
56 thread.deletedAt !== null ||
57 thread.settledOverride === "settled" ||
58 (thread.snoozedUntil != null && DateTime.isGreaterThan(thread.snoozedUntil, now))
59 )
60 return true;
61 const turn = latestProviderTurnForAttempt(projection.providerTurns, source.activeAttemptId);
62 if (
63 !settled &&
64 turn !== undefined &&
65 (turn.status === "completed" || turn.status === "failed" || turn.status === "interrupted")
66 )
67 return true;
68 const waitingRequest = projection.runtimeRequests.some(
69 (request) =>
70 request.kind !== "dynamic_tool_call" &&
71 request.kind !== "auth_refresh" &&
72 request.responseCapability.type !== "message" &&
73 (request.status === "pending" ||
74 ((request.status === "expired" || request.status === "cancelled") &&
75 request.responseCapability.type === "not_resumable" &&
76 turn !== undefined &&
77 turn.id === request.providerTurnId)),
78 );
79 if (waitingRequest) return true;
80 return projection.turnItems.some(
81 (item) =>
82 item.type === "secret_request" &&
83 item.runId === source.id &&
84 (item.secretStatus === "pending" ||
85 (item.secretStatus === "cancelled" &&
86 item.completedAt !== null &&
87 source.completedAt !== null &&
88 DateTime.Equivalence(item.completedAt, source.completedAt))),
89 );
90 }
91
92 /** A Stop from the user, or from the parent that delegated the task, that a restart cut short. */
93 export const restartStopRequested = Effect.fn("RestartContinuation.restartStopRequested")(
94 function* <E>(
95 getThreadRecords: <K extends ProjectionRecordField>(
96 threadId: ThreadId,
97 fields: ReadonlyArray<K>,
98 filter: ProjectionRecordFilter,
99 ) => Effect.Effect<ProjectionRecords<K>, E>,
100 thread: Pick<ProjectionRuntimeRecoveryState["thread"], "id" | "lineage" | "forkedFrom">,
101 source: Pick<OrchestrationV2Run, "id" | "delegatedTaskId">,
102 ) {
103 const { turnItems } = yield* getThreadRecords(thread.id, ["turnItems"], {
104 turnItemRunIds: [source.id],
105 turnItemTypes: ["run_interrupt_request"],
106 });
107 if (turnItems.some((item) => item.runId === source.id && item.type === "run_interrupt_request"))
108 return true;
109 const { lineage, forkedFrom } = thread;
110 const taskId =
111 source.delegatedTaskId ?? (forkedFrom?.type === "node" ? forkedFrom.nodeId : undefined);
112 if (
113 lineage.relationshipToParent !== "subagent" ||
114 lineage.parentThreadId === null ||
115 taskId === undefined
116 )
117 return false;
118 const task = (yield* getThreadRecords(lineage.parentThreadId, ["subagents"], {
119 subagentIds: [taskId],
120 })).subagents.find((candidate) => candidate.id === taskId);
121 // A settled task no longer owns later work in its thread.
122 if (
123 task === undefined ||
124 task.runId === null ||
125 task.result !== null ||
126 !isOrchestrationV2WorkActive(task.status)
127 )
128 return false;
129 return (
130 (yield* getThreadRecords(lineage.parentThreadId, ["turnItems"], {
131 turnItemTypes: ["run_interrupt_request"],
132 turnItemRunIds: [task.runId],
133 })).turnItems.length > 0
134 );
135 },
136 );
137
138 /** App-owned delegations continue in their own threads and report back. */
139 function hasOpenAwaitedWork(
140 projection: Pick<ProjectionRuntimeRecoveryState, "runs" | "turnItems">,
141 providerThread: ProjectionRuntimeRecoveryState["providerThreads"][number],
142 ): boolean {
143 const openItem = projection.turnItems.some((item) => {
144 if (item.status !== "pending" && item.status !== "running" && item.status !== "waiting")
145 return false;
146 if (item.type === "subagent" && item.origin === "app_owned" && item.childThreadId !== null)
147 return false;
148 const work = cancelledTurnItemWork(item);
149 return (
150 work !== undefined &&
151 isAwaitedWork(work) &&
152 awaitingProviderThreadId(item, projection.runs) === providerThread.id
153 );
154 });
155 return (
156 openItem ||
157 (providerThread.ownerNodeId === null &&
158 (providerThread.pendingBackgroundTasks ?? []).some((task) =>
159 isAwaitedWork(cancelledRosterTaskWork(task)),
160 ))
161 );
162 }
163
164 /** Settled runs require a live session so stale crash records cannot wake them. */
165 export function restartContinuationRun(
166 projection: Pick<
167 ProjectionRuntimeRecoveryState,
168 | "thread"
169 | "runs"
170 | "providerThreads"
171 | "providerSessions"
172 | "providerTurns"
173 | "runtimeRequests"
174 | "turnItems"
175 >,
176 now: DateTime.Utc,
177 ): OrchestrationV2Run | undefined {
178 const run = projection.runs.reduce<OrchestrationV2Run | undefined>(
179 (latest, candidate) =>
180 candidate.status !== "queued" &&
181 !neverStarted(candidate) &&
182 (!latest || runRanAfter(candidate, latest))
183 ? candidate
184 : latest,
185 undefined,
186 );
187 if (!run) return;
188 const preparedContinuation =
189 run.status === "starting" && run.restartContinuationOfRunId !== undefined;
190 const settled = run.status === "completed" || run.status === "waiting";
191 if (run.status !== "running" && !preparedContinuation && !settled) return;
192 if (restartContinuationBlocked(projection, run, now, settled)) return;
193 const liveTurnRequired = !preparedContinuation && !settled;
194 if (projection.thread.providerInstanceId !== run.providerInstanceId) return;
195 const providerThread = projection.providerThreads.find(
196 (thread) => thread.id === run.providerThreadId,
197 );
198 if (
199 !providerThread ||
200 providerThread.appThreadId !== projection.thread.id ||
201 providerThread.ownerNodeId !== null ||
202 providerThread.providerInstanceId !== run.providerInstanceId ||
203 providerThread.nativeThreadRef?.nativeId == null ||
204 providerThread.nativeThreadRef.strength !== "strong" ||
205 providerThread.nativeThreadRef.driver !== providerThread.driver ||
206 (liveTurnRequired && providerThread.status !== "active") ||
207 providerThread.status === "closed" ||
208 providerThread.status === "archived"
209 )
210 return;
211 const session = projection.providerSessions.find(
212 (candidate) => candidate.id === providerThread.providerSessionId,
213 );
214 // Adapters remain ready during turns; only stopped or failed sessions rule out liveness.
215 if (
216 session === undefined ||
217 session.providerInstanceId !== run.providerInstanceId ||
218 session.driver !== providerThread.driver ||
219 (!preparedContinuation && (session.status === "stopped" || session.status === "error"))
220 )
221 return;
222 if (settled) return hasOpenAwaitedWork(projection, providerThread) ? run : undefined;
223 if (
224 liveTurnRequired &&
225 !projection.providerTurns.some(
226 (turn) =>
227 turn.providerThreadId === providerThread.id &&
228 turn.runAttemptId === run.activeAttemptId &&
229 turn.status === "running",
230 )
231 )
232 return;
233 return run;
234 }
235
236 export const continueRestartedRun = Effect.fn("RestartContinuation.continueRestartedRun")(
237 function* (input: {
238 readonly threadId: ThreadId;
239 readonly sourceRunId: RunId;
240 readonly processLossCount?: number;
241 }) {
242 const settings = yield* ServerSettings.ServerSettingsService;
243 const enabled = yield* settings.getSettings.pipe(Effect.orElseSucceed(() => null));
244 if (!enabled) return;
245 const threads = yield* ThreadManagementService.ThreadManagementService;
246 const messageId = MessageId.make(`message:restart-continuation:${input.sourceRunId}`);
247 const projection = yield* threads.getThreadRecords(
248 input.threadId,
249 ["messages", "runs", "providerTurns", "attempts", "runtimeRequests", "turnItems"],
250 { messageIds: [messageId], turnItemTypes: ["secret_request"] },
251 );
252 if (
253 !resolveProjectSettings(enabled, projection.thread.projectId).settings
254 .continueThreadsAfterServerUpdate
255 )
256 return;
257 if (projection.messages.some((message) => message.id === messageId)) return;
258 const source = projection.runs.find((run) => run.id === input.sourceRunId);
259 if (
260 !source ||
261 !(
262 isRestartContinuationSource(source, projection.providerTurns) ||
263 continuationFailedBeforePrompt(source, projection.providerTurns)
264 ) ||
265 restartContinuationBlocked(
266 projection,
267 source,
268 yield* DateTime.now,
269 isRestartNoteSource(source, projection.providerTurns),
270 )
271 )
272 return;
273 if (
274 projection.runs.some(
275 (run) =>
276 run.id !== source.id &&
277 run.status !== "queued" &&
278 !cancelledBeforeStart(run) &&
279 runRanAfter(run, source),
280 )
281 )
282 return;
283 if (projection.thread.providerInstanceId !== source.providerInstanceId) return;
284 // A turn that keeps crashing the server would otherwise restart it forever.
285 let chainLength = 0;
286 const loopWindowStart =
287 DateTime.toEpochMillis(source.completedAt ?? source.requestedAt) -
288 CONTINUATION_LOOP_WINDOW_MS;
289 for (
290 let run: OrchestrationV2Run | undefined = source;
291 run?.restartContinuationOfRunId !== undefined && chainLength < MAX_CONTINUATION_CHAIN;
292 run = projection.runs.find((candidate) => candidate.id === run?.restartContinuationOfRunId)
293 ) {
294 if (DateTime.toEpochMillis(run.requestedAt) < loopWindowStart) break;
295 chainLength += 1;
296 }
297 if (
298 input.processLossCount === undefined
299 ? chainLength >= MAX_CONTINUATION_CHAIN
300 : input.processLossCount > MAX_CONTINUATION_CHAIN
301 ) {
302 yield* Effect.logWarning("Not continuing a run restarted too many times in a row", {
303 threadId: input.threadId,
304 sourceRunId: input.sourceRunId,
305 });
306 return;
307 }
308 if (yield* restartStopRequested(threads.getThreadRecords, projection.thread, source)) return;
309 const sourceMessage = (yield* threads.getThreadRecords(input.threadId, ["messages"], {
310 messageIds: [source.userMessageId],
311 })).messages.find((message) => message.id === source.userMessageId);
312 if (sourceMessage !== undefined && isNativeMaintenanceCommand(sourceMessage)) return;
313 const note = restartContinuationNote(
314 source,
315 projection.runs,
316 projection.providerTurns,
317 projection.attempts,
318 );
319 const noteText =
320 note.work.length === 0 ? undefined : restartCancelledBackgroundWorkNote(note.work);
321 const text =
322 noteText === undefined
323 ? CONTINUE_PROMPT
324 : note.settled
325 ? noteText
326 : `${noteText}\n\n${CONTINUE_PROMPT}`;
327 yield* threads
328 .dispatch({
329 type: "message.dispatch",
330 commandId: CommandId.make(`command:restart-continuation:${input.sourceRunId}`),
331 threadId: input.threadId,
332 messageId,
333 text,
334 notification: {
335 source: { kind: "system" },
336 outcome: "updated",
337 summary: note.settled
338 ? "T3 Code restarted and stopped background work"
339 : "T3 Code restarted and resumed this turn",
340 detail: text,
341 },
342 attachments: [],
343 modelSelection: source.modelSelection,
344 dispatchMode: { type: "start_immediately" },
345 createdBy: "agent",
346 creationSource: "server",
347 restartContinuationOfRunId: input.sourceRunId,
348 })
349 .pipe(
350 // Disabled instances, and threads above a policy ceiling, cannot resume; still run success
351 // cleanup so delegated children recover without futile retries.
352 Effect.catchTags({
353 ThreadManagementProviderInstanceDisabledError: (error) =>
354 Effect.logInfo("orchestration-v2.restart-continuation.instance-disabled", {
355 threadId: input.threadId,
356 sourceRunId: input.sourceRunId,
357 projectId: error.projectId,
358 instanceId: error.instanceId,
359 }),
360 OrchestratorRuntimeModeAboveCeilingError: () =>
361 Effect.logInfo("orchestration-v2.restart-continuation.above-ceiling", {
362 threadId: input.threadId,
363 sourceRunId: input.sourceRunId,
364 }),
365 }),
366 );
367 },
368 // Declined child continuations still owe parent results; existing continuation runs settle them instead.
369 (effect, input) =>
370 effect.pipe(
371 Effect.andThen(
372 Effect.gen(function* () {
373 const threads = yield* ThreadManagementService.ThreadManagementService;
374 yield* threads.recoverDelegatedTask(input.threadId, { sourceRunId: input.sourceRunId });
375 }),
376 ),
377 ),
378 );