The best way to control Claude Code, Codex and more from any computer or phone

main
ts 625 lines 22 KB
Raw
1 import {
2 CheckpointId,
3 CheckpointRef,
4 CheckpointScopeId,
5 NodeId,
6 OrchestrationV2Checkpoint,
7 OrchestrationV2CheckpointScope,
8 ProviderThreadId,
9 RunId,
10 ThreadId,
11 } from "@t3tools/contracts";
12 import * as Cache from "effect/Cache";
13 import * as Context from "effect/Context";
14 import * as Digest from "effect/crypto/Digest";
15 import * as DateTime from "effect/DateTime";
16 import * as Effect from "effect/Effect";
17 import * as Base64Url from "effect/encoding/Base64Url";
18 import * as Hex from "effect/encoding/Hex";
19 import * as KeyedLock from "@t3tools/shared/KeyedLock";
20 import * as Layer from "effect/Layer";
21 import * as Option from "effect/Option";
22 import * as Schema from "effect/Schema";
23
24 import { isGitImport, parseTurnDiffFilesFromNumstat } from "../checkpointing/Diffs.ts";
25 import * as CheckpointStore from "../checkpointing/CheckpointStore.ts";
26 import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
27
28 const CHECKPOINT_REFS_PREFIX = "refs/t3/orchestration-v2/checkpoints";
29 const ROOT_CHECKPOINT_SCOPE_NAME = "root";
30 // The provider turn waits on the baseline, so a workspace too slow for this starts without one.
31 // It bounds only the snapshot: waiting on the lock lets an in-flight capture finish the same ref.
32 const BASELINE_CAPTURE_BUDGET = "5 seconds";
33
34 export class CheckpointRootScopePrepareError extends Schema.TaggedError<CheckpointRootScopePrepareError>()(
35 "CheckpointRootScopePrepareError",
36 {
37 threadId: ThreadId,
38 runId: RunId,
39 cause: Schema.Defect(),
40 },
41 ) {
42 override get message(): string {
43 return `Failed to prepare root checkpoint scope for run ${this.runId}.`;
44 }
45 }
46
47 export class CheckpointBaselineCaptureError extends Schema.TaggedError<CheckpointBaselineCaptureError>()(
48 "CheckpointBaselineCaptureError",
49 {
50 scopeId: CheckpointScopeId,
51 ordinalWithinScope: Schema.Number,
52 cause: Schema.Defect(),
53 },
54 ) {
55 override get message(): string {
56 return `Failed to capture checkpoint baseline ${this.ordinalWithinScope} for scope ${this.scopeId}.`;
57 }
58 }
59
60 export class CheckpointCaptureError extends Schema.TaggedError<CheckpointCaptureError>()(
61 "CheckpointCaptureError",
62 {
63 scopeId: CheckpointScopeId,
64 parentCheckpointId: Schema.optional(CheckpointId),
65 cause: Schema.Defect(),
66 },
67 ) {
68 override get message(): string {
69 return `Failed to capture checkpoint for scope ${this.scopeId}.`;
70 }
71 }
72
73 export class CheckpointRestoreError extends Schema.TaggedError<CheckpointRestoreError>()(
74 "CheckpointRestoreError",
75 {
76 scopeId: CheckpointScopeId,
77 checkpointId: CheckpointId,
78 cause: Schema.Defect(),
79 },
80 ) {
81 override get message(): string {
82 return `Failed to restore checkpoint ${this.checkpointId} for scope ${this.scopeId}.`;
83 }
84 }
85
86 export class CheckpointDeleteStaleRefsError extends Schema.TaggedError<CheckpointDeleteStaleRefsError>()(
87 "CheckpointDeleteStaleRefsError",
88 {
89 scopeId: CheckpointScopeId,
90 checkpointIds: Schema.Array(CheckpointId),
91 cause: Schema.Defect(),
92 },
93 ) {
94 override get message(): string {
95 return `Failed to delete stale checkpoint refs for scope ${this.scopeId}.`;
96 }
97 }
98
99 export const CheckpointServiceV2Error = Schema.Union([
100 CheckpointRootScopePrepareError,
101 CheckpointBaselineCaptureError,
102 CheckpointCaptureError,
103 CheckpointRestoreError,
104 CheckpointDeleteStaleRefsError,
105 ]);
106 export type CheckpointServiceV2Error = typeof CheckpointServiceV2Error.Type;
107
108 const isCheckpointRestoreError = Schema.is(CheckpointRestoreError);
109
110 export interface CheckpointServiceV2Shape {
111 readonly prepareRootRunScope: (input: {
112 readonly threadId: ThreadId;
113 readonly runId: RunId;
114 readonly rootNodeId: NodeId;
115 readonly providerThreadId: ProviderThreadId;
116 readonly cwd: string;
117 readonly createdAt: DateTime.Utc;
118 }) => Effect.Effect<OrchestrationV2CheckpointScope, CheckpointServiceV2Error>;
119 readonly captureBaseline: (input: {
120 readonly scope: OrchestrationV2CheckpointScope;
121 readonly ordinalWithinScope: number;
122 }) => Effect.Effect<void, CheckpointServiceV2Error>;
123 readonly materializeBaselineCheckpoint: (input: {
124 readonly scope: OrchestrationV2CheckpointScope;
125 readonly ordinalWithinScope: number;
126 /** The run materializing the baseline: its root node, and when it was requested. */
127 readonly nodeId: NodeId;
128 readonly capturedAt: DateTime.Utc;
129 /** Records a missing checkpoint without touching git, for side chats sharing a parent's worktree. */
130 readonly forceMissing?: boolean;
131 }) => Effect.Effect<OrchestrationV2Checkpoint, CheckpointServiceV2Error>;
132 readonly capture: (input: {
133 readonly scope: OrchestrationV2CheckpointScope;
134 readonly runId: RunId | null;
135 readonly nodeId: NodeId;
136 readonly ordinalWithinScope: number;
137 readonly appRunOrdinal: number | null;
138 readonly capturedAt: DateTime.Utc;
139 readonly forceMissing?: boolean;
140 }) => Effect.Effect<OrchestrationV2Checkpoint, CheckpointServiceV2Error>;
141 readonly restore: (input: {
142 readonly scope: OrchestrationV2CheckpointScope;
143 readonly checkpoint: OrchestrationV2Checkpoint;
144 }) => Effect.Effect<void, CheckpointServiceV2Error>;
145 readonly deleteStaleRefs: (input: {
146 readonly scope: OrchestrationV2CheckpointScope;
147 readonly checkpoints: ReadonlyArray<OrchestrationV2Checkpoint>;
148 }) => Effect.Effect<void, CheckpointServiceV2Error>;
149 }
150
151 export class CheckpointServiceV2 extends Context.Service<
152 CheckpointServiceV2,
153 CheckpointServiceV2Shape
154 >()("t3/orchestration-v2/CheckpointService/CheckpointServiceV2") {}
155
156 export const checkpointRefForScopeOrdinal = Effect.fn("checkpointRefForScopeOrdinal")(
157 function* (input: { readonly scopeId: CheckpointScopeId; readonly ordinalWithinScope: number }) {
158 const crypto = yield* Digest.Digest;
159 const digest = yield* crypto
160 .digest(Digest.sha256, new TextEncoder().encode(input.scopeId))
161 .pipe(Effect.orDie);
162 const scopeKey = Hex.encode(digest).slice(0, 32);
163 return CheckpointRef.make(
164 `${CHECKPOINT_REFS_PREFIX}/${Base64Url.encode(scopeKey)}/ordinal/${input.ordinalWithinScope}`,
165 );
166 },
167 );
168
169 function checkpointIdForScopeOrdinal(
170 idAllocator: IdAllocator.IdAllocatorV2["Service"],
171 input: {
172 readonly scopeId: CheckpointScopeId;
173 readonly ordinalWithinScope: number;
174 },
175 ) {
176 return idAllocator.allocate.checkpoint({
177 checkpointScopeId: input.scopeId,
178 name: String(input.ordinalWithinScope),
179 });
180 }
181
182 const makeRootRunScope = Effect.fn("CheckpointService.makeRootRunScope")(function* (input: {
183 readonly idAllocator: IdAllocator.IdAllocatorV2["Service"];
184 readonly threadId: ThreadId;
185 readonly runId: RunId;
186 readonly rootNodeId: NodeId;
187 readonly providerThreadId: ProviderThreadId;
188 readonly cwd: string;
189 readonly createdAt: DateTime.Utc;
190 }) {
191 const scopeId = yield* input.idAllocator.allocate.checkpointScope({
192 threadId: input.threadId,
193 name: ROOT_CHECKPOINT_SCOPE_NAME,
194 });
195 return {
196 id: scopeId,
197 threadId: input.threadId,
198 runId: input.runId,
199 nodeId: input.rootNodeId,
200 parentScopeId: null,
201 providerThreadId: input.providerThreadId,
202 kind: "root_run",
203 ordinalWithinParent: 0,
204 advancesAppRunCount: true,
205 cwd: input.cwd,
206 createdAt: input.createdAt,
207 } satisfies OrchestrationV2CheckpointScope;
208 });
209
210 function makeCheckpoint(input: {
211 readonly id: CheckpointId;
212 readonly scope: OrchestrationV2CheckpointScope;
213 readonly runId: RunId | null;
214 readonly nodeId: NodeId;
215 readonly parentCheckpointId: CheckpointId | null;
216 readonly ordinalWithinScope: number;
217 readonly appRunOrdinal: number | null;
218 readonly ref: CheckpointRef;
219 readonly status: OrchestrationV2Checkpoint["status"];
220 readonly files: OrchestrationV2Checkpoint["files"];
221 readonly capturedAt: DateTime.Utc;
222 }): OrchestrationV2Checkpoint {
223 return {
224 id: input.id,
225 threadId: input.scope.threadId,
226 scopeId: input.scope.id,
227 runId: input.runId,
228 nodeId: input.nodeId,
229 parentCheckpointId: input.parentCheckpointId,
230 ordinalWithinScope: input.ordinalWithinScope,
231 appRunOrdinal: input.appRunOrdinal,
232 ref: input.ref,
233 status: input.status,
234 files: input.files,
235 capturedAt: input.capturedAt,
236 };
237 }
238
239 export const layer: Layer.Layer<
240 CheckpointServiceV2,
241 never,
242 CheckpointStore.CheckpointStore | Digest.Digest | IdAllocator.IdAllocatorV2
243 > = Layer.effect(
244 CheckpointServiceV2,
245 Effect.gen(function* () {
246 const checkpointStore = yield* CheckpointStore.CheckpointStore;
247 const crypto = yield* Digest.Digest;
248 const checkpointRefFor = (input: Parameters<typeof checkpointRefForScopeOrdinal>[0]) =>
249 checkpointRefForScopeOrdinal(input).pipe(Effect.provideService(Digest.Digest, crypto));
250 const idAllocator = yield* IdAllocator.IdAllocatorV2;
251 const workspaceLocks = yield* KeyedLock.make<string>();
252 const withWorkspaceLock = <A, E, R>(cwd: string, effect: Effect.Effect<A, E, R>) =>
253 workspaceLocks.withLock(cwd, effect);
254
255 const captureTimeouts = yield* Cache.make<string, number>({
256 capacity: 128,
257 lookup: () => Effect.succeed(0),
258 });
259 // Both callers hold the workspace lock; post-turn captures always probe for recovery.
260 const captureWorkspaceCheckpoint = Effect.fn("CheckpointService.captureWorkspaceCheckpoint")(
261 function* (input: CheckpointStore.CaptureCheckpointInput, baseline = false) {
262 const capture = checkpointStore.captureCheckpoint(input);
263 yield* (baseline ? capture.pipe(Effect.timeout(BASELINE_CAPTURE_BUDGET)) : capture).pipe(
264 Effect.tap(() => Cache.invalidate(captureTimeouts, input.cwd)),
265 Effect.tapError((error) =>
266 error._tag === "VcsProcessTimeoutError" || error._tag === "TimeoutError"
267 ? Effect.gen(function* () {
268 const previous = yield* Cache.get(captureTimeouts, input.cwd);
269 yield* Cache.set(captureTimeouts, input.cwd, previous + 1);
270 })
271 : Cache.invalidate(captureTimeouts, input.cwd),
272 ),
273 );
274 },
275 );
276
277 // A failed probe is an operational error, not proof that the workspace has no repository.
278 const probeRepository = (scope: OrchestrationV2CheckpointScope) =>
279 checkpointStore.isGitRepository(scope.cwd).pipe(
280 Effect.map((isRepository) =>
281 isRepository ? ("repository" as const) : ("missing" as const),
282 ),
283 Effect.catch((cause) =>
284 Effect.logWarning("orchestration V2 checkpoint repository probe failed", {
285 scopeId: scope.id,
286 cause: String(cause),
287 }).pipe(Effect.as("error" as const)),
288 ),
289 );
290
291 const captureBaseline: CheckpointServiceV2Shape["captureBaseline"] = (input) =>
292 withWorkspaceLock(
293 input.scope.cwd,
294 Effect.gen(function* () {
295 if (!(yield* checkpointStore.isGitRepository(input.scope.cwd))) {
296 return;
297 }
298
299 const checkpointRef = yield* checkpointRefFor({
300 scopeId: input.scope.id,
301 ordinalWithinScope: input.ordinalWithinScope,
302 });
303 const exists = yield* checkpointStore.hasCheckpointRef({
304 cwd: input.scope.cwd,
305 checkpointRef,
306 });
307 if (exists) {
308 return;
309 }
310
311 const timeouts = yield* Cache.getOption(captureTimeouts, input.scope.cwd);
312 if (Option.isSome(timeouts) && timeouts.value >= 2) {
313 return yield* Effect.logWarning(
314 "orchestration V2 checkpoint baseline skipped after repeated capture timeouts",
315 { scopeId: input.scope.id, ordinalWithinScope: input.ordinalWithinScope },
316 );
317 }
318
319 yield* captureWorkspaceCheckpoint({ cwd: input.scope.cwd, checkpointRef }, true).pipe(
320 Effect.catchTags({
321 VcsUnsupportedOperationError: () => Effect.void,
322 }),
323 );
324 }),
325 ).pipe(
326 Effect.mapError(
327 (cause) =>
328 new CheckpointBaselineCaptureError({
329 scopeId: input.scope.id,
330 ordinalWithinScope: input.ordinalWithinScope,
331 cause,
332 }),
333 ),
334 );
335
336 const materializeBaselineCheckpoint: CheckpointServiceV2Shape["materializeBaselineCheckpoint"] =
337 (input) =>
338 withWorkspaceLock(
339 input.scope.cwd,
340 Effect.gen(function* () {
341 const checkpointRef = yield* checkpointRefFor({
342 scopeId: input.scope.id,
343 ordinalWithinScope: input.ordinalWithinScope,
344 });
345 const checkpointId = yield* checkpointIdForScopeOrdinal(idAllocator, {
346 scopeId: input.scope.id,
347 ordinalWithinScope: input.ordinalWithinScope,
348 });
349 const repository =
350 input.forceMissing === true ? "missing" : yield* probeRepository(input.scope);
351 const status =
352 repository === "repository"
353 ? yield* checkpointStore
354 .hasCheckpointRef({
355 cwd: input.scope.cwd,
356 checkpointRef,
357 })
358 .pipe(
359 Effect.map((exists) => (exists ? ("ready" as const) : ("missing" as const))),
360 Effect.catch((cause) =>
361 Effect.logWarning("orchestration V2 baseline ref lookup failed", {
362 scopeId: input.scope.id,
363 checkpointRef,
364 cause: String(cause),
365 }).pipe(Effect.as("error" as const)),
366 ),
367 )
368 : repository;
369 return makeCheckpoint({
370 id: checkpointId,
371 scope: input.scope,
372 runId: null,
373 nodeId: input.nodeId,
374 parentCheckpointId: null,
375 ordinalWithinScope: input.ordinalWithinScope,
376 appRunOrdinal: null,
377 ref: checkpointRef,
378 status,
379 files: [],
380 capturedAt: input.capturedAt,
381 });
382 }),
383 ).pipe(
384 Effect.mapError(
385 (cause) =>
386 new CheckpointCaptureError({
387 scopeId: input.scope.id,
388 cause,
389 }),
390 ),
391 );
392
393 const capture: CheckpointServiceV2Shape["capture"] = (input) =>
394 withWorkspaceLock(
395 input.scope.cwd,
396 Effect.gen(function* () {
397 const checkpointId = yield* checkpointIdForScopeOrdinal(idAllocator, {
398 scopeId: input.scope.id,
399 ordinalWithinScope: input.ordinalWithinScope,
400 });
401 const parentCheckpointId =
402 input.ordinalWithinScope > 0
403 ? yield* checkpointIdForScopeOrdinal(idAllocator, {
404 scopeId: input.scope.id,
405 ordinalWithinScope: input.ordinalWithinScope - 1,
406 })
407 : null;
408 const checkpointRef = yield* checkpointRefFor({
409 scopeId: input.scope.id,
410 ordinalWithinScope: input.ordinalWithinScope,
411 });
412 const previousCheckpointRef = yield* checkpointRefFor({
413 scopeId: input.scope.id,
414 ordinalWithinScope: Math.max(0, input.ordinalWithinScope - 1),
415 });
416
417 const repository =
418 input.forceMissing === true ? "missing" : yield* probeRepository(input.scope);
419 if (repository !== "repository") {
420 return makeCheckpoint({
421 id: checkpointId,
422 scope: input.scope,
423 runId: input.runId,
424 nodeId: input.nodeId,
425 parentCheckpointId,
426 ordinalWithinScope: input.ordinalWithinScope,
427 appRunOrdinal: input.appRunOrdinal,
428 ref: checkpointRef,
429 status: repository,
430 files: [],
431 capturedAt: input.capturedAt,
432 });
433 }
434
435 const captureStatus = yield* captureWorkspaceCheckpoint({
436 cwd: input.scope.cwd,
437 checkpointRef,
438 ...(input.ordinalWithinScope > 0 ? { previousCheckpointRef } : {}),
439 }).pipe(
440 Effect.as("ready" as const),
441 Effect.catchTags({
442 VcsUnsupportedOperationError: () => Effect.succeed("missing" as const),
443 }),
444 Effect.catch((cause) =>
445 Effect.logWarning("orchestration V2 checkpoint capture failed", {
446 scopeId: input.scope.id,
447 checkpointRef,
448 cause: String(cause),
449 }).pipe(Effect.as("error" as const)),
450 ),
451 );
452
453 if (captureStatus !== "ready") {
454 return makeCheckpoint({
455 id: checkpointId,
456 scope: input.scope,
457 runId: input.runId,
458 nodeId: input.nodeId,
459 parentCheckpointId,
460 ordinalWithinScope: input.ordinalWithinScope,
461 appRunOrdinal: input.appRunOrdinal,
462 ref: checkpointRef,
463 status: captureStatus,
464 files: [],
465 capturedAt: input.capturedAt,
466 });
467 }
468
469 const previousExists = yield* checkpointStore
470 .hasCheckpointRef({
471 cwd: input.scope.cwd,
472 checkpointRef: previousCheckpointRef,
473 })
474 .pipe(
475 Effect.catch((cause) =>
476 Effect.logWarning("orchestration V2 previous checkpoint ref lookup failed", {
477 scopeId: input.scope.id,
478 checkpointRef: previousCheckpointRef,
479 cause: String(cause),
480 }).pipe(Effect.as(false)),
481 ),
482 );
483 const refs = {
484 cwd: input.scope.cwd,
485 fromCheckpointRef: previousCheckpointRef,
486 toCheckpointRef: checkpointRef,
487 };
488 // A pull or rebase can change thousands of files the turn did not write.
489 const files = previousExists
490 ? yield* Effect.all([
491 checkpointStore.diffCheckpoints({
492 ...refs,
493 fallbackFromToHead: false,
494 ignoreWhitespace: false,
495 format: "numstat",
496 }),
497 checkpointStore.listAuthoredPaths(refs).pipe(
498 Effect.catch((cause) =>
499 Effect.logWarning("orchestration V2 checkpoint authored paths failed", {
500 scopeId: input.scope.id,
501 checkpointRef,
502 cause: String(cause),
503 }).pipe(Effect.as(null)),
504 ),
505 ),
506 ]).pipe(
507 Effect.map(([diff, authoredPaths]) =>
508 parseTurnDiffFilesFromNumstat(diff)
509 .filter((file) => !isGitImport(file, authoredPaths))
510 .map((file) => ({
511 path: file.path,
512 kind: "modified",
513 additions: file.additions,
514 deletions: file.deletions,
515 })),
516 ),
517 Effect.catch((cause) =>
518 Effect.logWarning("orchestration V2 checkpoint diff summary failed", {
519 scopeId: input.scope.id,
520 checkpointRef,
521 cause: String(cause),
522 }).pipe(Effect.as([])),
523 ),
524 )
525 : [];
526
527 return makeCheckpoint({
528 id: checkpointId,
529 scope: input.scope,
530 runId: input.runId,
531 nodeId: input.nodeId,
532 parentCheckpointId,
533 ordinalWithinScope: input.ordinalWithinScope,
534 appRunOrdinal: input.appRunOrdinal,
535 ref: checkpointRef,
536 status: "ready",
537 files,
538 capturedAt: input.capturedAt,
539 });
540 }),
541 ).pipe(
542 Effect.mapError(
543 (cause) =>
544 new CheckpointCaptureError({
545 scopeId: input.scope.id,
546 cause,
547 }),
548 ),
549 );
550
551 const restore: CheckpointServiceV2Shape["restore"] = (input) =>
552 withWorkspaceLock(
553 input.scope.cwd,
554 Effect.gen(function* () {
555 if (input.checkpoint.status !== "ready") {
556 return yield* new CheckpointRestoreError({
557 scopeId: input.scope.id,
558 checkpointId: input.checkpoint.id,
559 cause: `Checkpoint status is ${input.checkpoint.status}.`,
560 });
561 }
562
563 const restored = yield* checkpointStore.restoreCheckpoint({
564 cwd: input.scope.cwd,
565 checkpointRef: input.checkpoint.ref,
566 fallbackToHead: false,
567 });
568 if (!restored) {
569 return yield* new CheckpointRestoreError({
570 scopeId: input.scope.id,
571 checkpointId: input.checkpoint.id,
572 cause: "Checkpoint ref is unavailable.",
573 });
574 }
575 }),
576 ).pipe(
577 Effect.mapError((cause) =>
578 isCheckpointRestoreError(cause)
579 ? cause
580 : new CheckpointRestoreError({
581 scopeId: input.scope.id,
582 checkpointId: input.checkpoint.id,
583 cause,
584 }),
585 ),
586 );
587
588 const deleteStaleRefs: CheckpointServiceV2Shape["deleteStaleRefs"] = (input) =>
589 withWorkspaceLock(
590 input.scope.cwd,
591 checkpointStore.deleteCheckpointRefs({
592 cwd: input.scope.cwd,
593 checkpointRefs: input.checkpoints.map((checkpoint) => checkpoint.ref),
594 }),
595 ).pipe(
596 Effect.mapError(
597 (cause) =>
598 new CheckpointDeleteStaleRefsError({
599 scopeId: input.scope.id,
600 checkpointIds: input.checkpoints.map((checkpoint) => checkpoint.id),
601 cause,
602 }),
603 ),
604 );
605
606 return CheckpointServiceV2.of({
607 prepareRootRunScope: (input) =>
608 makeRootRunScope({ ...input, idAllocator }).pipe(
609 Effect.mapError(
610 (cause) =>
611 new CheckpointRootScopePrepareError({
612 threadId: input.threadId,
613 runId: input.runId,
614 cause,
615 }),
616 ),
617 ),
618 captureBaseline,
619 materializeBaselineCheckpoint,
620 capture,
621 restore,
622 deleteStaleRefs,
623 } satisfies CheckpointServiceV2Shape);
624 }),
625 );