The best way to control Claude Code, Codex and more from any computer or phone

main
ts 357 lines 13.8 KB
Raw
1 import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
2 import { assert, it } from "@effect/vitest";
3 import {
4 CheckpointScopeId,
5 NodeId,
6 ProviderThreadId,
7 RunId,
8 ThreadId,
9 type OrchestrationV2CheckpointScope,
10 VcsProcessExitError,
11 VcsProcessTimeoutError,
12 VcsUnsupportedOperationError,
13 } from "@t3tools/contracts";
14 import * as Cause from "effect/Cause";
15 import * as DateTime from "effect/DateTime";
16 import * as Deferred from "effect/Deferred";
17 import * as Effect from "effect/Effect";
18 import * as Fiber from "effect/Fiber";
19 import * as Layer from "effect/Layer";
20 import * as Ref from "effect/Ref";
21 import * as TestClock from "effect/testing/TestClock";
22
23 import * as CheckpointStore from "../checkpointing/CheckpointStore.ts";
24 import * as CheckpointService from "./CheckpointService.ts";
25 import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
26
27 it.effect(
28 "skips unsupported baseline and turn checkpoints without swallowing capture failures",
29 () =>
30 Effect.gen(function* () {
31 const scope: OrchestrationV2CheckpointScope = {
32 id: CheckpointScopeId.make("checkpoint-scope:capture-failure"),
33 threadId: ThreadId.make("thread:capture-failure"),
34 runId: RunId.make("run:capture-failure:1"),
35 nodeId: NodeId.make("node:capture-failure:1"),
36 parentScopeId: null,
37 providerThreadId: ProviderThreadId.make("provider-thread:capture-failure"),
38 kind: "root_run",
39 ordinalWithinParent: 0,
40 advancesAppRunCount: true,
41 cwd: "/repo/ignored",
42 createdAt: DateTime.makeUnsafe("2026-07-28T00:00:00.000Z"),
43 };
44 for (const captureError of [
45 new VcsProcessTimeoutError({
46 operation: "test.capture",
47 command: "git",
48 cwd: scope.cwd,
49 timeoutMs: 30000,
50 }),
51 new VcsUnsupportedOperationError({
52 operation: "test.capture",
53 kind: "git",
54 detail: "The workspace directory is ignored by its Git repository.",
55 }),
56 ]) {
57 const layerTest = CheckpointService.layer.pipe(
58 Layer.provide(
59 Layer.mergeAll(
60 IdAllocator.layer,
61 Layer.mock(CheckpointStore.CheckpointStore)({
62 isGitRepository: () => Effect.succeed(true),
63 hasCheckpointRef: () => Effect.succeed(false),
64 captureCheckpoint: () => Effect.fail(captureError),
65 }),
66 ),
67 ),
68 Layer.provideMerge(NodeCrypto.layer),
69 );
70 yield* Effect.gen(function* () {
71 const checkpoints = yield* CheckpointService.CheckpointServiceV2;
72 const baseline = yield* Effect.result(
73 checkpoints.captureBaseline({ scope, ordinalWithinScope: 0 }),
74 );
75 const checkpoint = yield* checkpoints.capture({
76 scope,
77 ordinalWithinScope: 1,
78 runId: scope.runId!,
79 nodeId: scope.nodeId!,
80 appRunOrdinal: 1,
81 capturedAt: scope.createdAt,
82 });
83
84 assert.deepEqual(
85 { baseline: baseline._tag, turn: checkpoint.status, files: checkpoint.files },
86 captureError._tag === "VcsUnsupportedOperationError"
87 ? { baseline: "Success", turn: "missing", files: [] }
88 : { baseline: "Failure", turn: "error", files: [] },
89 );
90 if (baseline._tag === "Failure") {
91 assert.strictEqual(baseline.failure.cause, captureError);
92 } else {
93 const materialized = yield* checkpoints.materializeBaselineCheckpoint({
94 scope,
95 ordinalWithinScope: 0,
96 nodeId: scope.nodeId,
97 capturedAt: scope.createdAt,
98 });
99 assert.strictEqual(materialized.status, "missing");
100 }
101 }).pipe(Effect.provide(layerTest));
102 }
103 }),
104 );
105
106 it.effect(
107 "bounds a slow baseline snapshot but waits for an in-flight capture of the same ref",
108 () =>
109 Effect.gen(function* () {
110 const scope: OrchestrationV2CheckpointScope = {
111 id: CheckpointScopeId.make("checkpoint-scope:slow-baseline"),
112 threadId: ThreadId.make("thread:slow-baseline"),
113 runId: RunId.make("run:slow-baseline:1"),
114 nodeId: NodeId.make("node:slow-baseline:1"),
115 parentScopeId: null,
116 providerThreadId: ProviderThreadId.make("provider-thread:slow-baseline"),
117 kind: "root_run",
118 ordinalWithinParent: 0,
119 advancesAppRunCount: true,
120 cwd: "/repo/slow",
121 createdAt: DateTime.makeUnsafe("2026-07-28T00:00:00.000Z"),
122 };
123 const refs = yield* Ref.make<ReadonlySet<string>>(new Set());
124 const snapshots = yield* Ref.make(0);
125 const firstSnapshotStarted = yield* Deferred.make<void>();
126 const layerTest = CheckpointService.layer.pipe(
127 Layer.provide(
128 Layer.mergeAll(
129 IdAllocator.layer,
130 Layer.mock(CheckpointStore.CheckpointStore)({
131 isGitRepository: () => Effect.succeed(true),
132 hasCheckpointRef: ({ checkpointRef }) =>
133 Ref.get(refs).pipe(Effect.map((current) => current.has(checkpointRef))),
134 captureCheckpoint: ({ checkpointRef }) =>
135 Effect.gen(function* () {
136 yield* Ref.update(snapshots, (count) => count + 1);
137 yield* Deferred.succeed(firstSnapshotStarted, undefined);
138 yield* Effect.sleep("10 seconds");
139 yield* Ref.update(refs, (current) => new Set([...current, checkpointRef]));
140 }),
141 }),
142 ),
143 ),
144 Layer.provideMerge(NodeCrypto.layer),
145 );
146
147 yield* Effect.gen(function* () {
148 const checkpoints = yield* CheckpointService.CheckpointServiceV2;
149 // A stopped run's post-turn capture writes the next run's baseline ref while holding the lock.
150 const postTurn = yield* checkpoints
151 .capture({
152 scope,
153 ordinalWithinScope: 1,
154 runId: scope.runId!,
155 nodeId: scope.nodeId!,
156 appRunOrdinal: 1,
157 capturedAt: scope.createdAt,
158 })
159 .pipe(Effect.forkChild);
160 yield* Deferred.await(firstSnapshotStarted);
161 const waitingBaseline = yield* checkpoints
162 .captureBaseline({ scope, ordinalWithinScope: 1 })
163 .pipe(Effect.result, Effect.forkChild);
164 yield* TestClock.adjust("10 seconds");
165 assert.strictEqual((yield* Fiber.join(postTurn)).status, "ready");
166 assert.strictEqual((yield* Fiber.join(waitingBaseline))._tag, "Success");
167
168 for (const ordinalWithinScope of [2, 3]) {
169 const slowBaseline = yield* checkpoints
170 .captureBaseline({ scope, ordinalWithinScope })
171 .pipe(Effect.result, Effect.forkChild);
172 yield* TestClock.adjust("5 seconds");
173 const result = yield* Fiber.join(slowBaseline);
174 assert.strictEqual(result._tag, "Failure");
175 if (result._tag === "Failure") {
176 assert.isTrue(Cause.isTimeoutError(result.failure.cause));
177 }
178 }
179 const backedOffBaseline = yield* checkpoints
180 .captureBaseline({ scope, ordinalWithinScope: 4 })
181 .pipe(Effect.result, Effect.forkChild);
182 yield* TestClock.adjust("5 seconds");
183 assert.strictEqual((yield* Fiber.join(backedOffBaseline))._tag, "Success");
184 assert.strictEqual(yield* Ref.get(snapshots), 3);
185 assert.strictEqual((yield* Ref.get(refs)).size, 1);
186 }).pipe(Effect.provide(layerTest));
187 }),
188 );
189
190 it.effect("records a repository probe failure as an error checkpoint, not a missing one", () =>
191 Effect.gen(function* () {
192 const scope: OrchestrationV2CheckpointScope = {
193 id: CheckpointScopeId.make("checkpoint-scope:probe-failure"),
194 threadId: ThreadId.make("thread:probe-failure"),
195 runId: RunId.make("run:probe-failure:1"),
196 nodeId: NodeId.make("node:probe-failure:1"),
197 parentScopeId: null,
198 providerThreadId: ProviderThreadId.make("provider-thread:probe-failure"),
199 kind: "root_run",
200 ordinalWithinParent: 0,
201 advancesAppRunCount: true,
202 cwd: "/repo/bad-config",
203 createdAt: DateTime.makeUnsafe("2026-07-28T00:00:00.000Z"),
204 };
205 const layerTest = CheckpointService.layer.pipe(
206 Layer.provide(
207 Layer.mergeAll(
208 IdAllocator.layer,
209 Layer.mock(CheckpointStore.CheckpointStore)({
210 isGitRepository: () =>
211 Effect.fail(
212 new VcsProcessExitError({
213 operation: "GitVcsDriver.detectRepository",
214 command: "git",
215 cwd: scope.cwd,
216 exitCode: 128,
217 detail: "fatal: bad config line 1 in file .git/config",
218 }),
219 ),
220 }),
221 ),
222 ),
223 Layer.provideMerge(NodeCrypto.layer),
224 );
225
226 const statuses = yield* Effect.gen(function* () {
227 const checkpoints = yield* CheckpointService.CheckpointServiceV2;
228 const baseline = yield* checkpoints.materializeBaselineCheckpoint({
229 scope,
230 ordinalWithinScope: 0,
231 nodeId: scope.nodeId,
232 capturedAt: scope.createdAt,
233 });
234 const turn = yield* checkpoints.capture({
235 scope,
236 ordinalWithinScope: 1,
237 runId: scope.runId!,
238 nodeId: scope.nodeId!,
239 appRunOrdinal: 1,
240 capturedAt: scope.createdAt,
241 });
242 return { baseline: baseline.status, turn: turn.status };
243 }).pipe(Effect.provide(layerTest));
244
245 assert.deepEqual(statuses, { baseline: "error", turn: "error" });
246 }),
247 );
248
249 it.effect("backs off only baseline retries after repeated workspace capture timeouts", () =>
250 Effect.gen(function* () {
251 const scope: OrchestrationV2CheckpointScope = {
252 id: CheckpointScopeId.make("checkpoint-scope:slow-capture"),
253 threadId: ThreadId.make("thread:slow-capture"),
254 runId: RunId.make("run:slow-capture:1"),
255 nodeId: NodeId.make("node:slow-capture:1"),
256 parentScopeId: null,
257 providerThreadId: ProviderThreadId.make("provider-thread:slow-capture"),
258 kind: "root_run",
259 ordinalWithinParent: 0,
260 advancesAppRunCount: true,
261 cwd: "/repo/slow",
262 createdAt: DateTime.makeUnsafe("2026-07-28T00:00:00.000Z"),
263 };
264 const timeout = new VcsProcessTimeoutError({
265 operation: "GitVcsDriver.checkpoints.captureCheckpoint",
266 command: "git",
267 cwd: scope.cwd,
268 timeoutMs: 30_000,
269 });
270 let attempts = 0;
271 let healthy = false;
272 let nonTimeoutFailure = false;
273 const failure = new VcsProcessExitError({
274 operation: "test.capture",
275 command: "git",
276 cwd: scope.cwd,
277 exitCode: 128,
278 detail: "capture failed",
279 });
280 const layerTest = CheckpointService.layer.pipe(
281 Layer.provide(
282 Layer.mergeAll(
283 IdAllocator.layer,
284 Layer.mock(CheckpointStore.CheckpointStore)({
285 isGitRepository: () => Effect.succeed(true),
286 hasCheckpointRef: () => Effect.succeed(false),
287 captureCheckpoint: () =>
288 Effect.suspend(() => {
289 attempts++;
290 return healthy ? Effect.void : Effect.fail(nonTimeoutFailure ? failure : timeout);
291 }),
292 }),
293 ),
294 ),
295 Layer.provideMerge(NodeCrypto.layer),
296 );
297 yield* Effect.gen(function* () {
298 const checkpoints = yield* CheckpointService.CheckpointServiceV2;
299 const baseline = { scope, ordinalWithinScope: 0 };
300 const turn = {
301 scope,
302 ordinalWithinScope: 1,
303 runId: scope.runId,
304 nodeId: scope.nodeId,
305 appRunOrdinal: 1,
306 capturedAt: scope.createdAt,
307 };
308 yield* Effect.result(checkpoints.captureBaseline(baseline));
309 assert.strictEqual((yield* checkpoints.capture(turn)).status, "error");
310 assert.strictEqual(attempts, 2);
311
312 const otherThread = { ...scope, id: CheckpointScopeId.make("checkpoint-scope:other") };
313 const skipped = yield* Effect.result(
314 checkpoints.captureBaseline({ scope: otherThread, ordinalWithinScope: 0 }),
315 );
316 assert.strictEqual(attempts, 2, "baseline retried a repeatedly timing-out workspace");
317 assert.strictEqual(skipped._tag, "Success");
318
319 yield* Effect.result(
320 checkpoints.captureBaseline({
321 scope: { ...otherThread, cwd: "/repo/other" },
322 ordinalWithinScope: 0,
323 }),
324 );
325 assert.strictEqual(attempts, 3, "another workspace must still attempt capture");
326
327 const failedTurn = yield* checkpoints.capture(turn);
328 assert.strictEqual(failedTurn.status, "error");
329 assert.strictEqual(attempts, 4, "post-turn capture must not back off");
330
331 yield* checkpoints.captureBaseline(baseline);
332 assert.strictEqual(attempts, 4, "baseline retries must wait for post-turn recovery");
333 healthy = true;
334 assert.strictEqual((yield* checkpoints.capture(turn)).status, "ready");
335 assert.strictEqual(attempts, 5);
336 yield* checkpoints.captureBaseline(baseline);
337 assert.strictEqual(attempts, 6, "a successful post-turn capture must clear backoff");
338
339 healthy = false;
340 yield* Effect.result(checkpoints.captureBaseline(baseline));
341 yield* Effect.result(checkpoints.captureBaseline(baseline));
342 yield* checkpoints.captureBaseline(baseline);
343 assert.strictEqual(attempts, 8);
344 nonTimeoutFailure = true;
345 assert.strictEqual((yield* checkpoints.capture(turn)).status, "error");
346 nonTimeoutFailure = false;
347 yield* Effect.result(checkpoints.captureBaseline(baseline));
348 assert.strictEqual(attempts, 10, "a non-timeout failure must clear backoff");
349 healthy = true;
350 yield* checkpoints.captureBaseline(baseline);
351 healthy = false;
352 yield* Effect.result(checkpoints.captureBaseline(baseline));
353 yield* Effect.result(checkpoints.captureBaseline(baseline));
354 assert.strictEqual(attempts, 13, "a successful baseline must clear the timeout streak");
355 }).pipe(Effect.provide(layerTest));
356 }),
357 );