The best way to control Claude Code, Codex and more from any computer or phone

main
ts 505 lines 19.5 KB
Raw
1 import type {
2 NodeId,
3 OrchestrationV2ConversationMessage,
4 OrchestrationV2PendingBackgroundTask,
5 OrchestrationV2ProviderThread,
6 OrchestrationV2ProviderTurn,
7 OrchestrationV2RestartCancelledBackgroundWork,
8 OrchestrationV2Run,
9 OrchestrationV2RunAttempt,
10 OrchestrationV2Subagent,
11 OrchestrationV2TurnItem,
12 } from "@t3tools/contracts";
13 import { subagentHoldsCompletion } from "@t3tools/provider-core/server/subagentProjection";
14 import { backgroundWorkHoldsCompletion } from "@t3tools/shared/orchestrationV2PendingBackgroundWork";
15 import { latestRootProviderFailure, runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError";
16
17 type Work = OrchestrationV2RestartCancelledBackgroundWork;
18 type Attempt = Pick<OrchestrationV2RunAttempt, "id" | "runId">;
19
20 const MAX_LABEL_LENGTH = 160;
21
22 function compactLabel(value: unknown, maxLength = MAX_LABEL_LENGTH): string | undefined {
23 if (typeof value !== "string") return undefined;
24 const text = value.replaceAll(/\s+/g, " ").trim();
25 if (text.length === 0) return undefined;
26 if (text.length <= maxLength) return text;
27 // Half an emoji serializes to a lone surrogate escape, which provider JSON parsers reject.
28 const last = text.charCodeAt(maxLength - 2);
29 const end = last >= 0xd800 && last <= 0xdbff ? maxLength - 2 : maxLength - 1;
30 return `${text.slice(0, end)}…`;
31 }
32
33 /** A bounded label that keeps its id suffix whole whenever it fits. */
34 function labelWithId(name: unknown, fallback: string, suffix: string): string {
35 const budget = MAX_LABEL_LENGTH - suffix.length;
36 if (budget < 2) return compactLabel(`${fallback}${suffix}`) ?? fallback;
37 return `${compactLabel(name, budget) ?? compactLabel(fallback, budget)}${suffix}`;
38 }
39
40 /** resumeId overrides the item id for native resume, such as Codex's child provider-thread id. */
41 export function cancelledTurnItemWork(
42 item: OrchestrationV2TurnItem,
43 resumeId?: string,
44 ): Work | undefined {
45 switch (item.type) {
46 case "subagent": {
47 const nativeId =
48 item.origin === "provider_native" ? (resumeId ?? item.nativeItemRef?.nativeId) : undefined;
49 return {
50 kind: "subagent",
51 label: labelWithId(
52 compactLabel(item.title) === undefined ? item.prompt : item.title,
53 "subagent",
54 nativeId == null
55 ? ""
56 : item.nativeItemRef?.driver === "claudeAgent"
57 ? ` (id ${nativeId}; resume it with SendMessage to '${nativeId}')`
58 : ` (id ${nativeId})`,
59 ),
60 id: item.id,
61 };
62 }
63 case "command_execution":
64 return {
65 kind: item.monitor === true ? "monitor" : "shell",
66 label: compactLabel(item.input) ?? compactLabel(item.title) ?? "background command",
67 id: item.id,
68 ...(item.completionWake === "always" ? { awaited: true as const } : {}),
69 };
70 case "dynamic_tool": {
71 const monitor =
72 item.input !== null &&
73 typeof item.input === "object" &&
74 Reflect.get(item.input, "persistent") === true;
75 return {
76 kind: monitor ? "monitor" : "task",
77 label: compactLabel(item.title) ?? compactLabel(item.toolName) ?? "background tool",
78 id: item.id,
79 };
80 }
81 default:
82 return undefined;
83 }
84 }
85
86 /** Returns the awaiting provider thread; native subagents can run on children while their launching turn awaits them. */
87 export function awaitingProviderThreadId(
88 item: OrchestrationV2TurnItem,
89 runs: ReadonlyArray<Pick<OrchestrationV2Run, "id" | "providerThreadId">>,
90 ): OrchestrationV2Run["providerThreadId"] | undefined {
91 const runProviderThreadId = runs.find((run) => run.id === item.runId)?.providerThreadId;
92 return item.type === "subagent" && item.origin === "provider_native"
93 ? (runProviderThreadId ?? item.providerThreadId)
94 : (item.providerThreadId ?? runProviderThreadId);
95 }
96
97 function restartWorkKind(task: OrchestrationV2PendingBackgroundTask): Work["kind"] {
98 switch (task.kind) {
99 case "command":
100 return "shell";
101 case "subagent":
102 return "subagent";
103 case "monitor":
104 return "monitor";
105 case "background_task":
106 return "task";
107 default:
108 task satisfies never;
109 return "task";
110 }
111 }
112
113 export function cancelledRosterTaskWork(task: OrchestrationV2PendingBackgroundTask): Work {
114 if (task.parentTaskId !== undefined) {
115 return {
116 kind: "subagent",
117 label: labelWithId(task.description, "background task", ` (subagent ${task.parentTaskId})`),
118 id: task.parentTaskId,
119 };
120 }
121 const kind = restartWorkKind(task);
122 const description = compactLabel(task.description);
123 return {
124 kind,
125 label:
126 compactLabel(
127 description === undefined ? task.taskId : `${description} (id ${task.taskId})`,
128 ) ?? "background task",
129 id: task.taskId,
130 ...(task.awaited === true ? { awaited: true as const } : {}),
131 };
132 }
133
134 const MAX_NOTE_ENTRIES = 10;
135
136 /** Bounds entries and labels to protect the context budget. */
137 export function restartCancelledBackgroundWorkNote(work: ReadonlyArray<Work>): string {
138 const omitted = work.length - MAX_NOTE_ENTRIES;
139 return [
140 "Note: the T3 server restarted, and this background work was cancelled before it finished. It will not report back on its own, so resume or restart what you still need and wait for its result:",
141 ...work.slice(0, MAX_NOTE_ENTRIES).map((entry) => `- ${entry.kind}: ${entry.label}`),
142 ...(omitted > 0 ? [`- and ${omitted} more`] : []),
143 ].join("\n");
144 }
145
146 type ProviderTurnState = Pick<
147 OrchestrationV2ProviderTurn,
148 "runAttemptId" | "providerThreadId" | "status"
149 >;
150
151 export function isRestartNoteSource(
152 source: Pick<OrchestrationV2Run, "status" | "activeAttemptId" | "restartCancelledBackgroundWork">,
153 providerTurns: ReadonlyArray<Pick<ProviderTurnState, "runAttemptId" | "status">>,
154 ): boolean {
155 return isNoteSource(source, (attemptId) =>
156 providerTurns.some((turn) => turn.runAttemptId === attemptId && turn.status === "cancelled"),
157 );
158 }
159
160 function isNoteSource(
161 source: Pick<OrchestrationV2Run, "status" | "activeAttemptId" | "restartCancelledBackgroundWork">,
162 attemptCancelled: (attemptId: OrchestrationV2Run["activeAttemptId"]) => boolean,
163 ): boolean {
164 return (
165 // Failed and interrupted turns never continue; recovery-cancelled waiting runs still qualify.
166 (source.status === "completed" ||
167 source.status === "waiting" ||
168 source.status === "cancelled") &&
169 (source.restartCancelledBackgroundWork?.length ?? 0) > 0 &&
170 !attemptCancelled(source.activeAttemptId)
171 );
172 }
173
174 /**
175 * Matches the Working-section hold: a shell or monitor wakes it only when the agent said it
176 * waits on it. Most are dev servers and their watchdogs, so their note waits for the next
177 * user message.
178 */
179 export function isAwaitedWork(work: Pick<Work, "kind" | "awaited">): boolean {
180 return work.awaited === true || (work.kind !== "shell" && work.kind !== "monitor");
181 }
182
183 export function runOutcome(
184 run: Pick<OrchestrationV2Run, "status" | "restartCancelledBackgroundWork">,
185 ): OrchestrationV2Run["status"] {
186 return (run.status === "completed" || run.status === "cancelled") &&
187 (run.restartCancelledBackgroundWork ?? []).some(isAwaitedWork)
188 ? "interrupted"
189 : run.status;
190 }
191
192 /** Unstarted runs have no lost work to continue, except prepared continuations carrying a source run. */
193 export function neverStarted(
194 run: Pick<OrchestrationV2Run, "startedAt" | "restartContinuationOfRunId">,
195 ): boolean {
196 return run.startedAt === null && run.restartContinuationOfRunId === undefined;
197 }
198
199 /** Restart-cancelled unstarted runs supersede nothing; user-stopped unstarted runs do. */
200 export function cancelledBeforeStart(
201 run: Pick<OrchestrationV2Run, "status" | "startedAt" | "restartContinuationOfRunId">,
202 ): boolean {
203 return run.status === "cancelled" && neverStarted(run);
204 }
205
206 /** A follow-up task's runs: those tagged with it, and the untagged wakes for subagents they started. */
207 export function delegatedTaskRuns(
208 projection: {
209 readonly runs: ReadonlyArray<OrchestrationV2Run>;
210 readonly messages: ReadonlyArray<
211 Pick<OrchestrationV2ConversationMessage, "runId" | "delegatedCompletion">
212 >;
213 },
214 taskId: NodeId,
215 ): ReadonlyArray<OrchestrationV2Run> {
216 const owned = new Set(
217 projection.runs.filter((run) => run.delegatedTaskId === taskId).map((run) => run.id),
218 );
219 const untagged = new Set(
220 projection.runs.filter((run) => run.delegatedTaskId === undefined).map((run) => run.id),
221 );
222 for (let grew = owned.size > 0; grew;) {
223 grew = false;
224 for (const { runId, delegatedCompletion } of projection.messages) {
225 if (
226 runId === null ||
227 !untagged.has(runId) ||
228 owned.has(runId) ||
229 delegatedCompletion === undefined ||
230 !owned.has(delegatedCompletion.parentRunId)
231 )
232 continue;
233 owned.add(runId);
234 grew = true;
235 }
236 }
237 return projection.runs.filter((run) => owned.has(run.id));
238 }
239
240 /**
241 * A follow-up task's runs still owe a subagent they started its result, or still await
242 * background work they started, as the original task waits on both.
243 */
244 export function delegatedTaskAwaitsWork(
245 runs: ReadonlyArray<OrchestrationV2Run>,
246 child: {
247 readonly subagents: ReadonlyArray<
248 Pick<OrchestrationV2Subagent, "runId" | "status" | "completionDelivery">
249 >;
250 readonly providerThreads: ReadonlyArray<
251 Pick<OrchestrationV2ProviderThread, "pendingBackgroundTasks">
252 >;
253 },
254 ): boolean {
255 const runIds = new Set<string | null>(runs.map((run) => run.id));
256 return (
257 child.subagents.some((task) => runIds.has(task.runId) && subagentHoldsCompletion(task)) ||
258 child.providerThreads.some((thread) =>
259 backgroundWorkHoldsCompletion(
260 (thread.pendingBackgroundTasks ?? []).filter(
261 (task) => task.runId !== undefined && runIds.has(task.runId),
262 ),
263 ),
264 )
265 );
266 }
267
268 /** A follow-up task's latest run, passing over queued work it no longer waits on, as the original task does. */
269 export function delegatedTaskResultRun(
270 runs: ReadonlyArray<OrchestrationV2Run>,
271 ): OrchestrationV2Run | undefined {
272 const taskRuns = runs.toSorted((a, b) => b.ordinal - a.ordinal);
273 const awaited = taskRuns.filter(
274 (run) => !cancelledBeforeStart(run) && !(run.status === "queued" && run.queueHeld === true),
275 );
276 // Executing work reports the task's state; a reordered queue can finish a newer run before an older one.
277 const executing = awaited.find((run) =>
278 ["preparing", "starting", "running", "waiting"].includes(run.status),
279 );
280 return executing ?? awaited.find((run) => run.status === "queued") ?? awaited[0] ?? taskRuns[0];
281 }
282
283 /** The provider never accepted this continuation's prompt, so starting it again repeats nothing. */
284 export function continuationFailedBeforePrompt(
285 run: Pick<OrchestrationV2Run, "status" | "activeAttemptId" | "restartContinuationOfRunId">,
286 providerTurns: ReadonlyArray<Pick<ProviderTurnState, "runAttemptId">>,
287 ): boolean {
288 return (
289 run.status === "failed" &&
290 run.restartContinuationOfRunId !== undefined &&
291 !providerTurns.some((turn) => turn.runAttemptId === run.activeAttemptId)
292 );
293 }
294
295 const CONTINUATION_START_RETRY_DELAYS_MS = [60_000, 180_000];
296
297 /** Backoff before restarting a continuation that failed to start, or undefined to let it fail. */
298 export function continuationStartRetryDelayMs(
299 run: OrchestrationV2Run,
300 projection: {
301 readonly runs: ReadonlyArray<OrchestrationV2Run>;
302 readonly providerTurns: ReadonlyArray<Pick<ProviderTurnState, "runAttemptId">>;
303 readonly turnItems: Parameters<typeof latestRootProviderFailure>[1];
304 },
305 ): number | undefined {
306 if (!continuationFailedBeforePrompt(run, projection.providerTurns)) return;
307 // These fail the same way however often they start.
308 const failureClass = latestRootProviderFailure(run, projection.turnItems)?.class;
309 if (
310 failureClass === "usage_limit" ||
311 failureClass === "permission_error" ||
312 failureClass === "validation_error"
313 )
314 return;
315 let failedStarts = 0;
316 for (
317 let current: OrchestrationV2Run | undefined = run;
318 current !== undefined && continuationFailedBeforePrompt(current, projection.providerTurns);
319 current = projection.runs.find(
320 (candidate) => candidate.id === current?.restartContinuationOfRunId,
321 )
322 )
323 failedStarts += 1;
324 return CONTINUATION_START_RETRY_DELAYS_MS[failedStarts - 1];
325 }
326
327 export function isRestartContinuationSource(
328 source: Pick<OrchestrationV2Run, "status" | "activeAttemptId" | "restartCancelledBackgroundWork">,
329 providerTurns: ReadonlyArray<Pick<ProviderTurnState, "runAttemptId" | "status">>,
330 ): boolean {
331 return isRestartNoteSource(source, providerTurns)
332 ? (source.restartCancelledBackgroundWork ?? []).some(isAwaitedWork)
333 : source.status === "cancelled";
334 }
335
336 /** Built once per call so long threads are read a bounded number of times, not once per run. */
337 function continuationLookup(
338 runs: ReadonlyArray<OrchestrationV2Run>,
339 providerTurns: ReadonlyArray<Pick<ProviderTurnState, "runAttemptId" | "status">>,
340 attempts: ReadonlyArray<Attempt>,
341 ) {
342 const runsById = new Map<string, OrchestrationV2Run>();
343 for (const run of runs) if (!runsById.has(run.id)) runsById.set(run.id, run);
344 const completedAttempts = new Set<string | null>();
345 const cancelledAttempts = new Set<string | null>();
346 for (const turn of providerTurns) {
347 if (turn.status === "completed") completedAttempts.add(turn.runAttemptId);
348 else if (turn.status === "cancelled") cancelledAttempts.add(turn.runAttemptId);
349 }
350 const completedRuns = new Set<string>();
351 for (const attempt of attempts)
352 if (completedAttempts.has(attempt.id)) completedRuns.add(attempt.runId);
353 return { runsById, completedAttempts, cancelledAttempts, completedRuns };
354 }
355
356 type ContinuationLookup = ReturnType<typeof continuationLookup>;
357
358 /** Running does not prove prompt acceptance; carry continuation notes until a completed turn proves delivery. */
359 export function restartContinuationNote(
360 source: OrchestrationV2Run,
361 runs: ReadonlyArray<OrchestrationV2Run>,
362 providerTurns: ReadonlyArray<Pick<ProviderTurnState, "runAttemptId" | "status">>,
363 attempts: ReadonlyArray<Attempt>,
364 ): { readonly work: ReadonlyArray<Work>; readonly settled: boolean } {
365 return continuationNote(source, continuationLookup(runs, providerTurns, attempts));
366 }
367
368 function continuationNote(
369 source: OrchestrationV2Run,
370 { runsById, completedAttempts, cancelledAttempts, completedRuns }: ContinuationLookup,
371 ): { readonly work: ReadonlyArray<Work>; readonly settled: boolean } {
372 let current = source;
373 let work = current.restartCancelledBackgroundWork ?? [];
374 const visited = new Set([current.id]);
375 while (
376 current.restartContinuationOfRunId !== undefined &&
377 !completedRuns.has(current.id) &&
378 !(current.activeAttemptId !== null && completedAttempts.has(current.activeAttemptId))
379 ) {
380 const previous = runsById.get(current.restartContinuationOfRunId);
381 if (previous === undefined || visited.has(previous.id)) break;
382 visited.add(previous.id);
383 current = previous;
384 work = mergeRestartCancelledBackgroundWork(current.restartCancelledBackgroundWork ?? [], work);
385 }
386 return {
387 work,
388 settled: isNoteSource(current, (attemptId) => cancelledAttempts.has(attemptId)),
389 };
390 }
391
392 export function isRestartNoteContinuation(
393 run: Pick<OrchestrationV2Run, "restartContinuationOfRunId">,
394 runs: ReadonlyArray<OrchestrationV2Run>,
395 providerTurns: ReadonlyArray<Pick<ProviderTurnState, "runAttemptId" | "status">>,
396 attempts: ReadonlyArray<Attempt>,
397 ): boolean {
398 return (
399 run.restartContinuationOfRunId !== undefined &&
400 isNoteContinuation(run, continuationLookup(runs, providerTurns, attempts))
401 );
402 }
403
404 function isNoteContinuation(
405 run: Pick<OrchestrationV2Run, "restartContinuationOfRunId">,
406 lookup: ContinuationLookup,
407 ): boolean {
408 const source =
409 run.restartContinuationOfRunId === undefined
410 ? undefined
411 : lookup.runsById.get(run.restartContinuationOfRunId);
412 return source !== undefined && continuationNote(source, lookup).work.length > 0;
413 }
414
415 /** Derive delivery from later completed turns on the same provider thread, excluding compactions, resumes, and rolled-back runs. */
416 export function pendingRestartCancelledBackgroundWork(input: {
417 readonly runs: ReadonlyArray<OrchestrationV2Run>;
418 readonly providerTurns: ReadonlyArray<ProviderTurnState>;
419 /** Called only when an earlier run on the provider thread carries restart-note work. */
420 readonly compactionMessageIds: () => ReadonlySet<string>;
421 readonly run: Pick<
422 OrchestrationV2Run,
423 | "id"
424 | "ordinal"
425 | "completedAt"
426 | "userMessageId"
427 | "providerThreadId"
428 | "restartContinuationOfRunId"
429 | "activeAttemptId"
430 >;
431
432 readonly attempts: ReadonlyArray<Attempt>;
433 }): ReadonlyArray<Work> {
434 if (input.run.providerThreadId === null || input.run.restartContinuationOfRunId !== undefined)
435 return [];
436 const providerThreadId = input.run.providerThreadId;
437 const sameThread = input.runs.filter((run) => run.providerThreadId === providerThreadId);
438 const sources = sameThread.filter(
439 (source) =>
440 (source.restartCancelledBackgroundWork?.length ?? 0) > 0 && runRanAfter(input.run, source),
441 );
442 // Threads whose server never restarted carry no notes; skip every other lookup for them.
443 if (sources.length === 0) return [];
444 const compactionMessageIds = input.compactionMessageIds();
445 const isCompaction = (run: typeof input.run) => compactionMessageIds.has(run.userMessageId);
446 if (isCompaction(input.run)) return [];
447 const deliveredAttemptIds = new Set(
448 input.providerTurns
449 .filter((turn) => turn.providerThreadId === providerThreadId && turn.status === "completed")
450 .map((turn) => turn.runAttemptId),
451 );
452 // Earlier attempts may already have delivered the note; a steer's replacement attempt must not repeat it.
453 const alreadyDelivered = input.attempts.some(
454 (attempt) =>
455 attempt.runId === input.run.id &&
456 attempt.id !== input.run.activeAttemptId &&
457 deliveredAttemptIds.has(attempt.id),
458 );
459 if (alreadyDelivered) return [];
460 const deliveredRunIds = new Set<string>();
461 for (const attempt of input.attempts)
462 if (deliveredAttemptIds.has(attempt.id)) deliveredRunIds.add(attempt.runId);
463 let lookup: ContinuationLookup | undefined;
464 // runRanAfter is a total order, so a source precedes some prompted run iff it precedes the latest one.
465 let latestPrompted: OrchestrationV2Run | undefined;
466 for (const candidate of sameThread) {
467 if (
468 candidate.id !== input.run.id &&
469 candidate.activeAttemptId !== null &&
470 candidate.status !== "rolled_back" &&
471 (deliveredAttemptIds.has(candidate.activeAttemptId) || deliveredRunIds.has(candidate.id)) &&
472 !isCompaction(candidate) &&
473 (candidate.restartContinuationOfRunId === undefined ||
474 isNoteContinuation(
475 candidate,
476 (lookup ??= continuationLookup(input.runs, input.providerTurns, input.attempts)),
477 )) &&
478 (latestPrompted === undefined || runRanAfter(candidate, latestPrompted))
479 )
480 latestPrompted = candidate;
481 }
482 return sources
483 .filter((source) => latestPrompted === undefined || !runRanAfter(latestPrompted, source))
484 .reduce<ReadonlyArray<Work>>(
485 (work, source) =>
486 mergeRestartCancelledBackgroundWork(work, source.restartCancelledBackgroundWork ?? []),
487 [],
488 );
489 }
490
491 export function mergeRestartCancelledBackgroundWork(
492 current: ReadonlyArray<Work>,
493 added: ReadonlyArray<Work>,
494 ): ReadonlyArray<Work> {
495 const identity = (entry: Work) => entry.id ?? `${entry.kind}\u0000${entry.label}`;
496 const seen = new Set(current.map(identity));
497 const merged = [...current];
498 for (const entry of added) {
499 const key = identity(entry);
500 if (seen.has(key)) continue;
501 seen.add(key);
502 merged.push(entry);
503 }
504 return merged;
505 }