The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; |
| 2 | import { assert, it } from "@effect/vitest"; |
| 3 | import { |
| 4 | CheckpointScopeId, |
| 5 | NodeId, |
| 6 | ProviderThreadId, |
| 7 | RunId, |
| 8 | ThreadId, |
| 9 | type OrchestrationV2CheckpointScope, |
| 10 | VcsProcessExitError, |
| 11 | VcsProcessTimeoutError, |
| 12 | VcsUnsupportedOperationError, |
| 13 | } from "@t3tools/contracts"; |
| 14 | import * as Cause from "effect/Cause"; |
| 15 | import * as DateTime from "effect/DateTime"; |
| 16 | import * as Deferred from "effect/Deferred"; |
| 17 | import * as Effect from "effect/Effect"; |
| 18 | import * as Fiber from "effect/Fiber"; |
| 19 | import * as Layer from "effect/Layer"; |
| 20 | import * as Ref from "effect/Ref"; |
| 21 | import * as TestClock from "effect/testing/TestClock"; |
| 22 | |
| 23 | import * as CheckpointStore from "../checkpointing/CheckpointStore.ts"; |
| 24 | import * as CheckpointService from "./CheckpointService.ts"; |
| 25 | import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; |
| 26 | |
| 27 | it.effect( |
| 28 | "skips unsupported baseline and turn checkpoints without swallowing capture failures", |
| 29 | () => |
| 30 | Effect.gen(function* () { |
| 31 | const scope: OrchestrationV2CheckpointScope = { |
| 32 | id: CheckpointScopeId.make("checkpoint-scope:capture-failure"), |
| 33 | threadId: ThreadId.make("thread:capture-failure"), |
| 34 | runId: RunId.make("run:capture-failure:1"), |
| 35 | nodeId: NodeId.make("node:capture-failure:1"), |
| 36 | parentScopeId: null, |
| 37 | providerThreadId: ProviderThreadId.make("provider-thread:capture-failure"), |
| 38 | kind: "root_run", |
| 39 | ordinalWithinParent: 0, |
| 40 | advancesAppRunCount: true, |
| 41 | cwd: "/repo/ignored", |
| 42 | createdAt: DateTime.makeUnsafe("2026-07-28T00:00:00.000Z"), |
| 43 | }; |
| 44 | for (const captureError of [ |
| 45 | new VcsProcessTimeoutError({ |
| 46 | operation: "test.capture", |
| 47 | command: "git", |
| 48 | cwd: scope.cwd, |
| 49 | timeoutMs: 30000, |
| 50 | }), |
| 51 | new VcsUnsupportedOperationError({ |
| 52 | operation: "test.capture", |
| 53 | kind: "git", |
| 54 | detail: "The workspace directory is ignored by its Git repository.", |
| 55 | }), |
| 56 | ]) { |
| 57 | const layerTest = CheckpointService.layer.pipe( |
| 58 | Layer.provide( |
| 59 | Layer.mergeAll( |
| 60 | IdAllocator.layer, |
| 61 | Layer.mock(CheckpointStore.CheckpointStore)({ |
| 62 | isGitRepository: () => Effect.succeed(true), |
| 63 | hasCheckpointRef: () => Effect.succeed(false), |
| 64 | captureCheckpoint: () => Effect.fail(captureError), |
| 65 | }), |
| 66 | ), |
| 67 | ), |
| 68 | Layer.provideMerge(NodeCrypto.layer), |
| 69 | ); |
| 70 | yield* Effect.gen(function* () { |
| 71 | const checkpoints = yield* CheckpointService.CheckpointServiceV2; |
| 72 | const baseline = yield* Effect.result( |
| 73 | checkpoints.captureBaseline({ scope, ordinalWithinScope: 0 }), |
| 74 | ); |
| 75 | const checkpoint = yield* checkpoints.capture({ |
| 76 | scope, |
| 77 | ordinalWithinScope: 1, |
| 78 | runId: scope.runId!, |
| 79 | nodeId: scope.nodeId!, |
| 80 | appRunOrdinal: 1, |
| 81 | capturedAt: scope.createdAt, |
| 82 | }); |
| 83 | |
| 84 | assert.deepEqual( |
| 85 | { baseline: baseline._tag, turn: checkpoint.status, files: checkpoint.files }, |
| 86 | captureError._tag === "VcsUnsupportedOperationError" |
| 87 | ? { baseline: "Success", turn: "missing", files: [] } |
| 88 | : { baseline: "Failure", turn: "error", files: [] }, |
| 89 | ); |
| 90 | if (baseline._tag === "Failure") { |
| 91 | assert.strictEqual(baseline.failure.cause, captureError); |
| 92 | } else { |
| 93 | const materialized = yield* checkpoints.materializeBaselineCheckpoint({ |
| 94 | scope, |
| 95 | ordinalWithinScope: 0, |
| 96 | nodeId: scope.nodeId, |
| 97 | capturedAt: scope.createdAt, |
| 98 | }); |
| 99 | assert.strictEqual(materialized.status, "missing"); |
| 100 | } |
| 101 | }).pipe(Effect.provide(layerTest)); |
| 102 | } |
| 103 | }), |
| 104 | ); |
| 105 | |
| 106 | it.effect( |
| 107 | "bounds a slow baseline snapshot but waits for an in-flight capture of the same ref", |
| 108 | () => |
| 109 | Effect.gen(function* () { |
| 110 | const scope: OrchestrationV2CheckpointScope = { |
| 111 | id: CheckpointScopeId.make("checkpoint-scope:slow-baseline"), |
| 112 | threadId: ThreadId.make("thread:slow-baseline"), |
| 113 | runId: RunId.make("run:slow-baseline:1"), |
| 114 | nodeId: NodeId.make("node:slow-baseline:1"), |
| 115 | parentScopeId: null, |
| 116 | providerThreadId: ProviderThreadId.make("provider-thread:slow-baseline"), |
| 117 | kind: "root_run", |
| 118 | ordinalWithinParent: 0, |
| 119 | advancesAppRunCount: true, |
| 120 | cwd: "/repo/slow", |
| 121 | createdAt: DateTime.makeUnsafe("2026-07-28T00:00:00.000Z"), |
| 122 | }; |
| 123 | const refs = yield* Ref.make<ReadonlySet<string>>(new Set()); |
| 124 | const snapshots = yield* Ref.make(0); |
| 125 | const firstSnapshotStarted = yield* Deferred.make<void>(); |
| 126 | const layerTest = CheckpointService.layer.pipe( |
| 127 | Layer.provide( |
| 128 | Layer.mergeAll( |
| 129 | IdAllocator.layer, |
| 130 | Layer.mock(CheckpointStore.CheckpointStore)({ |
| 131 | isGitRepository: () => Effect.succeed(true), |
| 132 | hasCheckpointRef: ({ checkpointRef }) => |
| 133 | Ref.get(refs).pipe(Effect.map((current) => current.has(checkpointRef))), |
| 134 | captureCheckpoint: ({ checkpointRef }) => |
| 135 | Effect.gen(function* () { |
| 136 | yield* Ref.update(snapshots, (count) => count + 1); |
| 137 | yield* Deferred.succeed(firstSnapshotStarted, undefined); |
| 138 | yield* Effect.sleep("10 seconds"); |
| 139 | yield* Ref.update(refs, (current) => new Set([...current, checkpointRef])); |
| 140 | }), |
| 141 | }), |
| 142 | ), |
| 143 | ), |
| 144 | Layer.provideMerge(NodeCrypto.layer), |
| 145 | ); |
| 146 | |
| 147 | yield* Effect.gen(function* () { |
| 148 | const checkpoints = yield* CheckpointService.CheckpointServiceV2; |
| 149 | // A stopped run's post-turn capture writes the next run's baseline ref while holding the lock. |
| 150 | const postTurn = yield* checkpoints |
| 151 | .capture({ |
| 152 | scope, |
| 153 | ordinalWithinScope: 1, |
| 154 | runId: scope.runId!, |
| 155 | nodeId: scope.nodeId!, |
| 156 | appRunOrdinal: 1, |
| 157 | capturedAt: scope.createdAt, |
| 158 | }) |
| 159 | .pipe(Effect.forkChild); |
| 160 | yield* Deferred.await(firstSnapshotStarted); |
| 161 | const waitingBaseline = yield* checkpoints |
| 162 | .captureBaseline({ scope, ordinalWithinScope: 1 }) |
| 163 | .pipe(Effect.result, Effect.forkChild); |
| 164 | yield* TestClock.adjust("10 seconds"); |
| 165 | assert.strictEqual((yield* Fiber.join(postTurn)).status, "ready"); |
| 166 | assert.strictEqual((yield* Fiber.join(waitingBaseline))._tag, "Success"); |
| 167 | |
| 168 | for (const ordinalWithinScope of [2, 3]) { |
| 169 | const slowBaseline = yield* checkpoints |
| 170 | .captureBaseline({ scope, ordinalWithinScope }) |
| 171 | .pipe(Effect.result, Effect.forkChild); |
| 172 | yield* TestClock.adjust("5 seconds"); |
| 173 | const result = yield* Fiber.join(slowBaseline); |
| 174 | assert.strictEqual(result._tag, "Failure"); |
| 175 | if (result._tag === "Failure") { |
| 176 | assert.isTrue(Cause.isTimeoutError(result.failure.cause)); |
| 177 | } |
| 178 | } |
| 179 | const backedOffBaseline = yield* checkpoints |
| 180 | .captureBaseline({ scope, ordinalWithinScope: 4 }) |
| 181 | .pipe(Effect.result, Effect.forkChild); |
| 182 | yield* TestClock.adjust("5 seconds"); |
| 183 | assert.strictEqual((yield* Fiber.join(backedOffBaseline))._tag, "Success"); |
| 184 | assert.strictEqual(yield* Ref.get(snapshots), 3); |
| 185 | assert.strictEqual((yield* Ref.get(refs)).size, 1); |
| 186 | }).pipe(Effect.provide(layerTest)); |
| 187 | }), |
| 188 | ); |
| 189 | |
| 190 | it.effect("records a repository probe failure as an error checkpoint, not a missing one", () => |
| 191 | Effect.gen(function* () { |
| 192 | const scope: OrchestrationV2CheckpointScope = { |
| 193 | id: CheckpointScopeId.make("checkpoint-scope:probe-failure"), |
| 194 | threadId: ThreadId.make("thread:probe-failure"), |
| 195 | runId: RunId.make("run:probe-failure:1"), |
| 196 | nodeId: NodeId.make("node:probe-failure:1"), |
| 197 | parentScopeId: null, |
| 198 | providerThreadId: ProviderThreadId.make("provider-thread:probe-failure"), |
| 199 | kind: "root_run", |
| 200 | ordinalWithinParent: 0, |
| 201 | advancesAppRunCount: true, |
| 202 | cwd: "/repo/bad-config", |
| 203 | createdAt: DateTime.makeUnsafe("2026-07-28T00:00:00.000Z"), |
| 204 | }; |
| 205 | const layerTest = CheckpointService.layer.pipe( |
| 206 | Layer.provide( |
| 207 | Layer.mergeAll( |
| 208 | IdAllocator.layer, |
| 209 | Layer.mock(CheckpointStore.CheckpointStore)({ |
| 210 | isGitRepository: () => |
| 211 | Effect.fail( |
| 212 | new VcsProcessExitError({ |
| 213 | operation: "GitVcsDriver.detectRepository", |
| 214 | command: "git", |
| 215 | cwd: scope.cwd, |
| 216 | exitCode: 128, |
| 217 | detail: "fatal: bad config line 1 in file .git/config", |
| 218 | }), |
| 219 | ), |
| 220 | }), |
| 221 | ), |
| 222 | ), |
| 223 | Layer.provideMerge(NodeCrypto.layer), |
| 224 | ); |
| 225 | |
| 226 | const statuses = yield* Effect.gen(function* () { |
| 227 | const checkpoints = yield* CheckpointService.CheckpointServiceV2; |
| 228 | const baseline = yield* checkpoints.materializeBaselineCheckpoint({ |
| 229 | scope, |
| 230 | ordinalWithinScope: 0, |
| 231 | nodeId: scope.nodeId, |
| 232 | capturedAt: scope.createdAt, |
| 233 | }); |
| 234 | const turn = yield* checkpoints.capture({ |
| 235 | scope, |
| 236 | ordinalWithinScope: 1, |
| 237 | runId: scope.runId!, |
| 238 | nodeId: scope.nodeId!, |
| 239 | appRunOrdinal: 1, |
| 240 | capturedAt: scope.createdAt, |
| 241 | }); |
| 242 | return { baseline: baseline.status, turn: turn.status }; |
| 243 | }).pipe(Effect.provide(layerTest)); |
| 244 | |
| 245 | assert.deepEqual(statuses, { baseline: "error", turn: "error" }); |
| 246 | }), |
| 247 | ); |
| 248 | |
| 249 | it.effect("backs off only baseline retries after repeated workspace capture timeouts", () => |
| 250 | Effect.gen(function* () { |
| 251 | const scope: OrchestrationV2CheckpointScope = { |
| 252 | id: CheckpointScopeId.make("checkpoint-scope:slow-capture"), |
| 253 | threadId: ThreadId.make("thread:slow-capture"), |
| 254 | runId: RunId.make("run:slow-capture:1"), |
| 255 | nodeId: NodeId.make("node:slow-capture:1"), |
| 256 | parentScopeId: null, |
| 257 | providerThreadId: ProviderThreadId.make("provider-thread:slow-capture"), |
| 258 | kind: "root_run", |
| 259 | ordinalWithinParent: 0, |
| 260 | advancesAppRunCount: true, |
| 261 | cwd: "/repo/slow", |
| 262 | createdAt: DateTime.makeUnsafe("2026-07-28T00:00:00.000Z"), |
| 263 | }; |
| 264 | const timeout = new VcsProcessTimeoutError({ |
| 265 | operation: "GitVcsDriver.checkpoints.captureCheckpoint", |
| 266 | command: "git", |
| 267 | cwd: scope.cwd, |
| 268 | timeoutMs: 30_000, |
| 269 | }); |
| 270 | let attempts = 0; |
| 271 | let healthy = false; |
| 272 | let nonTimeoutFailure = false; |
| 273 | const failure = new VcsProcessExitError({ |
| 274 | operation: "test.capture", |
| 275 | command: "git", |
| 276 | cwd: scope.cwd, |
| 277 | exitCode: 128, |
| 278 | detail: "capture failed", |
| 279 | }); |
| 280 | const layerTest = CheckpointService.layer.pipe( |
| 281 | Layer.provide( |
| 282 | Layer.mergeAll( |
| 283 | IdAllocator.layer, |
| 284 | Layer.mock(CheckpointStore.CheckpointStore)({ |
| 285 | isGitRepository: () => Effect.succeed(true), |
| 286 | hasCheckpointRef: () => Effect.succeed(false), |
| 287 | captureCheckpoint: () => |
| 288 | Effect.suspend(() => { |
| 289 | attempts++; |
| 290 | return healthy ? Effect.void : Effect.fail(nonTimeoutFailure ? failure : timeout); |
| 291 | }), |
| 292 | }), |
| 293 | ), |
| 294 | ), |
| 295 | Layer.provideMerge(NodeCrypto.layer), |
| 296 | ); |
| 297 | yield* Effect.gen(function* () { |
| 298 | const checkpoints = yield* CheckpointService.CheckpointServiceV2; |
| 299 | const baseline = { scope, ordinalWithinScope: 0 }; |
| 300 | const turn = { |
| 301 | scope, |
| 302 | ordinalWithinScope: 1, |
| 303 | runId: scope.runId, |
| 304 | nodeId: scope.nodeId, |
| 305 | appRunOrdinal: 1, |
| 306 | capturedAt: scope.createdAt, |
| 307 | }; |
| 308 | yield* Effect.result(checkpoints.captureBaseline(baseline)); |
| 309 | assert.strictEqual((yield* checkpoints.capture(turn)).status, "error"); |
| 310 | assert.strictEqual(attempts, 2); |
| 311 | |
| 312 | const otherThread = { ...scope, id: CheckpointScopeId.make("checkpoint-scope:other") }; |
| 313 | const skipped = yield* Effect.result( |
| 314 | checkpoints.captureBaseline({ scope: otherThread, ordinalWithinScope: 0 }), |
| 315 | ); |
| 316 | assert.strictEqual(attempts, 2, "baseline retried a repeatedly timing-out workspace"); |
| 317 | assert.strictEqual(skipped._tag, "Success"); |
| 318 | |
| 319 | yield* Effect.result( |
| 320 | checkpoints.captureBaseline({ |
| 321 | scope: { ...otherThread, cwd: "/repo/other" }, |
| 322 | ordinalWithinScope: 0, |
| 323 | }), |
| 324 | ); |
| 325 | assert.strictEqual(attempts, 3, "another workspace must still attempt capture"); |
| 326 | |
| 327 | const failedTurn = yield* checkpoints.capture(turn); |
| 328 | assert.strictEqual(failedTurn.status, "error"); |
| 329 | assert.strictEqual(attempts, 4, "post-turn capture must not back off"); |
| 330 | |
| 331 | yield* checkpoints.captureBaseline(baseline); |
| 332 | assert.strictEqual(attempts, 4, "baseline retries must wait for post-turn recovery"); |
| 333 | healthy = true; |
| 334 | assert.strictEqual((yield* checkpoints.capture(turn)).status, "ready"); |
| 335 | assert.strictEqual(attempts, 5); |
| 336 | yield* checkpoints.captureBaseline(baseline); |
| 337 | assert.strictEqual(attempts, 6, "a successful post-turn capture must clear backoff"); |
| 338 | |
| 339 | healthy = false; |
| 340 | yield* Effect.result(checkpoints.captureBaseline(baseline)); |
| 341 | yield* Effect.result(checkpoints.captureBaseline(baseline)); |
| 342 | yield* checkpoints.captureBaseline(baseline); |
| 343 | assert.strictEqual(attempts, 8); |
| 344 | nonTimeoutFailure = true; |
| 345 | assert.strictEqual((yield* checkpoints.capture(turn)).status, "error"); |
| 346 | nonTimeoutFailure = false; |
| 347 | yield* Effect.result(checkpoints.captureBaseline(baseline)); |
| 348 | assert.strictEqual(attempts, 10, "a non-timeout failure must clear backoff"); |
| 349 | healthy = true; |
| 350 | yield* checkpoints.captureBaseline(baseline); |
| 351 | healthy = false; |
| 352 | yield* Effect.result(checkpoints.captureBaseline(baseline)); |
| 353 | yield* Effect.result(checkpoints.captureBaseline(baseline)); |
| 354 | assert.strictEqual(attempts, 13, "a successful baseline must clear the timeout streak"); |
| 355 | }).pipe(Effect.provide(layerTest)); |
| 356 | }), |
| 357 | ); |