The best way to control Claude Code, Codex and more from any computer or phone
fix(server): repeated capture timeouts no longer delay turn starts (#29196)
Co-authored-by: maria-rcks <maria@kuuro.net>
Lakshmi Tanmay committed
Nov 20, 2026 at 01:32 UTC
61085457d76ea246b5955c092ffba2452a825aff
2 files changed
+179
-33
apps/server/src/orchestration-v2/CheckpointService.test.ts
+126
-4
@@ -11,6 +11,7 @@ import {
11
VcsProcessTimeoutError,
12
VcsUnsupportedOperationError,
13
} from "@t3tools/contracts";
14
+import * as Cause from "effect/Cause";
15
import * as DateTime from "effect/DateTime";
16
import * as Deferred from "effect/Deferred";
17
import * as Effect from "effect/Effect";
@@ -164,12 +165,23 @@ it.effect(
165
assert.strictEqual((yield* Fiber.join(postTurn)).status, "ready");
166
assert.strictEqual((yield* Fiber.join(waitingBaseline))._tag, "Success");
167
167
- const slowBaseline = yield* checkpoints
168
- .captureBaseline({ scope, ordinalWithinScope: 2 })
168
+ for (const ordinalWithinScope of [2, 3]) {
169
+ const slowBaseline = yield* checkpoints
170
+ .captureBaseline({ scope, ordinalWithinScope })
171
+ .pipe(Effect.result, Effect.forkChild);
172
+ yield* TestClock.adjust("5 seconds");
173
+ const result = yield* Fiber.join(slowBaseline);
174
+ assert.strictEqual(result._tag, "Failure");
175
+ if (result._tag === "Failure") {
176
+ assert.isTrue(Cause.isTimeoutError(result.failure.cause));
177
+ }
178
+ }
179
+ const backedOffBaseline = yield* checkpoints
180
+ .captureBaseline({ scope, ordinalWithinScope: 4 })
181
.pipe(Effect.result, Effect.forkChild);
182
yield* TestClock.adjust("5 seconds");
171
- assert.strictEqual((yield* Fiber.join(slowBaseline))._tag, "Failure");
172
- assert.strictEqual(yield* Ref.get(snapshots), 2);
183
+ assert.strictEqual((yield* Fiber.join(backedOffBaseline))._tag, "Success");
184
+ assert.strictEqual(yield* Ref.get(snapshots), 3);
185
assert.strictEqual((yield* Ref.get(refs)).size, 1);
186
}).pipe(Effect.provide(layerTest));
187
}),
@@ -233,3 +245,113 @@ it.effect("records a repository probe failure as an error checkpoint, not a miss
245
assert.deepEqual(statuses, { baseline: "error", turn: "error" });
246
}),
247
);
248
+
249
+it.effect("backs off only baseline retries after repeated workspace capture timeouts", () =>
250
+ Effect.gen(function* () {
251
+ const scope: OrchestrationV2CheckpointScope = {
252
+ id: CheckpointScopeId.make("checkpoint-scope:slow-capture"),
253
+ threadId: ThreadId.make("thread:slow-capture"),
254
+ runId: RunId.make("run:slow-capture:1"),
255
+ nodeId: NodeId.make("node:slow-capture:1"),
256
+ parentScopeId: null,
257
+ providerThreadId: ProviderThreadId.make("provider-thread:slow-capture"),
258
+ kind: "root_run",
259
+ ordinalWithinParent: 0,
260
+ advancesAppRunCount: true,
261
+ cwd: "/repo/slow",
262
+ createdAt: DateTime.makeUnsafe("2026-07-28T00:00:00.000Z"),
263
+ };
264
+ const timeout = new VcsProcessTimeoutError({
265
+ operation: "GitVcsDriver.checkpoints.captureCheckpoint",
266
+ command: "git",
267
+ cwd: scope.cwd,
268
+ timeoutMs: 30_000,
269
+ });
270
+ let attempts = 0;
271
+ let healthy = false;
272
+ let nonTimeoutFailure = false;
273
+ const failure = new VcsProcessExitError({
274
+ operation: "test.capture",
275
+ command: "git",
276
+ cwd: scope.cwd,
277
+ exitCode: 128,
278
+ detail: "capture failed",
279
+ });
280
+ const layerTest = CheckpointService.layer.pipe(
281
+ Layer.provide(
282
+ Layer.mergeAll(
283
+ IdAllocator.layer,
284
+ Layer.mock(CheckpointStore.CheckpointStore)({
285
+ isGitRepository: () => Effect.succeed(true),
286
+ hasCheckpointRef: () => Effect.succeed(false),
287
+ captureCheckpoint: () =>
288
+ Effect.suspend(() => {
289
+ attempts++;
290
+ return healthy ? Effect.void : Effect.fail(nonTimeoutFailure ? failure : timeout);
291
+ }),
292
+ }),
293
+ ),
294
+ ),
295
+ Layer.provideMerge(NodeCrypto.layer),
296
+ );
297
+ yield* Effect.gen(function* () {
298
+ const checkpoints = yield* CheckpointService.CheckpointServiceV2;
299
+ const baseline = { scope, ordinalWithinScope: 0 };
300
+ const turn = {
301
+ scope,
302
+ ordinalWithinScope: 1,
303
+ runId: scope.runId,
304
+ nodeId: scope.nodeId,
305
+ appRunOrdinal: 1,
306
+ capturedAt: scope.createdAt,
307
+ };
308
+ yield* Effect.result(checkpoints.captureBaseline(baseline));
309
+ assert.strictEqual((yield* checkpoints.capture(turn)).status, "error");
310
+ assert.strictEqual(attempts, 2);
311
+
312
+ const otherThread = { ...scope, id: CheckpointScopeId.make("checkpoint-scope:other") };
313
+ const skipped = yield* Effect.result(
314
+ checkpoints.captureBaseline({ scope: otherThread, ordinalWithinScope: 0 }),
315
+ );
316
+ assert.strictEqual(attempts, 2, "baseline retried a repeatedly timing-out workspace");
317
+ assert.strictEqual(skipped._tag, "Success");
318
+
319
+ yield* Effect.result(
320
+ checkpoints.captureBaseline({
321
+ scope: { ...otherThread, cwd: "/repo/other" },
322
+ ordinalWithinScope: 0,
323
+ }),
324
+ );
325
+ assert.strictEqual(attempts, 3, "another workspace must still attempt capture");
326
+
327
+ const failedTurn = yield* checkpoints.capture(turn);
328
+ assert.strictEqual(failedTurn.status, "error");
329
+ assert.strictEqual(attempts, 4, "post-turn capture must not back off");
330
+
331
+ yield* checkpoints.captureBaseline(baseline);
332
+ assert.strictEqual(attempts, 4, "baseline retries must wait for post-turn recovery");
333
+ healthy = true;
334
+ assert.strictEqual((yield* checkpoints.capture(turn)).status, "ready");
335
+ assert.strictEqual(attempts, 5);
336
+ yield* checkpoints.captureBaseline(baseline);
337
+ assert.strictEqual(attempts, 6, "a successful post-turn capture must clear backoff");
338
+
339
+ healthy = false;
340
+ yield* Effect.result(checkpoints.captureBaseline(baseline));
341
+ yield* Effect.result(checkpoints.captureBaseline(baseline));
342
+ yield* checkpoints.captureBaseline(baseline);
343
+ assert.strictEqual(attempts, 8);
344
+ nonTimeoutFailure = true;
345
+ assert.strictEqual((yield* checkpoints.capture(turn)).status, "error");
346
+ nonTimeoutFailure = false;
347
+ yield* Effect.result(checkpoints.captureBaseline(baseline));
348
+ assert.strictEqual(attempts, 10, "a non-timeout failure must clear backoff");
349
+ healthy = true;
350
+ yield* checkpoints.captureBaseline(baseline);
351
+ healthy = false;
352
+ yield* Effect.result(checkpoints.captureBaseline(baseline));
353
+ yield* Effect.result(checkpoints.captureBaseline(baseline));
354
+ assert.strictEqual(attempts, 13, "a successful baseline must clear the timeout streak");
355
+ }).pipe(Effect.provide(layerTest));
356
+ }),
357
+);
apps/server/src/orchestration-v2/CheckpointService.ts
+53
-29
@@ -9,6 +9,7 @@ import {
9
RunId,
10
ThreadId,
11
} from "@t3tools/contracts";
12
+import * as Cache from "effect/Cache";
13
import * as Context from "effect/Context";
14
import * as Digest from "effect/crypto/Digest";
15
import * as DateTime from "effect/DateTime";
@@ -17,6 +18,7 @@ import * as Base64Url from "effect/encoding/Base64Url";
18
import * as Hex from "effect/encoding/Hex";
19
import * as KeyedLock from "@t3tools/shared/KeyedLock";
20
import * as Layer from "effect/Layer";
21
+import * as Option from "effect/Option";
22
import * as Schema from "effect/Schema";
23
24
import { isGitImport, parseTurnDiffFilesFromNumstat } from "../checkpointing/Diffs.ts";
@@ -250,6 +252,28 @@ export const layer: Layer.Layer<
252
const withWorkspaceLock = <A, E, R>(cwd: string, effect: Effect.Effect<A, E, R>) =>
253
workspaceLocks.withLock(cwd, effect);
254
255
+ const captureTimeouts = yield* Cache.make<string, number>({
256
+ capacity: 128,
257
+ lookup: () => Effect.succeed(0),
258
+ });
259
+ // Both callers hold the workspace lock; post-turn captures always probe for recovery.
260
+ const captureWorkspaceCheckpoint = Effect.fn("CheckpointService.captureWorkspaceCheckpoint")(
261
+ function* (input: CheckpointStore.CaptureCheckpointInput, baseline = false) {
262
+ const capture = checkpointStore.captureCheckpoint(input);
263
+ yield* (baseline ? capture.pipe(Effect.timeout(BASELINE_CAPTURE_BUDGET)) : capture).pipe(
264
+ Effect.tap(() => Cache.invalidate(captureTimeouts, input.cwd)),
265
+ Effect.tapError((error) =>
266
+ error._tag === "VcsProcessTimeoutError" || error._tag === "TimeoutError"
267
+ ? Effect.gen(function* () {
268
+ const previous = yield* Cache.get(captureTimeouts, input.cwd);
269
+ yield* Cache.set(captureTimeouts, input.cwd, previous + 1);
270
+ })
271
+ : Cache.invalidate(captureTimeouts, input.cwd),
272
+ ),
273
+ );
274
+ },
275
+ );
276
+
277
// A failed probe is an operational error, not proof that the workspace has no repository.
278
const probeRepository = (scope: OrchestrationV2CheckpointScope) =>
279
checkpointStore.isGitRepository(scope.cwd).pipe(
@@ -284,17 +308,19 @@ export const layer: Layer.Layer<
308
return;
309
}
310
287
- yield* checkpointStore
288
- .captureCheckpoint({
289
- cwd: input.scope.cwd,
290
- checkpointRef,
291
- })
292
- .pipe(
293
- Effect.timeout(BASELINE_CAPTURE_BUDGET),
294
- Effect.catchTags({
295
- VcsUnsupportedOperationError: () => Effect.void,
296
- }),
311
+ const timeouts = yield* Cache.getOption(captureTimeouts, input.scope.cwd);
312
+ if (Option.isSome(timeouts) && timeouts.value >= 2) {
313
+ return yield* Effect.logWarning(
314
+ "orchestration V2 checkpoint baseline skipped after repeated capture timeouts",
315
+ { scopeId: input.scope.id, ordinalWithinScope: input.ordinalWithinScope },
316
);
317
+ }
318
+
319
+ yield* captureWorkspaceCheckpoint({ cwd: input.scope.cwd, checkpointRef }, true).pipe(
320
+ Effect.catchTags({
321
+ VcsUnsupportedOperationError: () => Effect.void,
322
+ }),
323
+ );
324
}),
325
).pipe(
326
Effect.mapError(
@@ -406,25 +432,23 @@ export const layer: Layer.Layer<
432
});
433
}
434
409
- const captureStatus = yield* checkpointStore
410
- .captureCheckpoint({
411
- cwd: input.scope.cwd,
412
- checkpointRef,
413
- ...(input.ordinalWithinScope > 0 ? { previousCheckpointRef } : {}),
414
- })
415
- .pipe(
416
- Effect.as("ready" as const),
417
- Effect.catchTags({
418
- VcsUnsupportedOperationError: () => Effect.succeed("missing" as const),
419
- }),
420
- Effect.catch((cause) =>
421
- Effect.logWarning("orchestration V2 checkpoint capture failed", {
422
- scopeId: input.scope.id,
423
- checkpointRef,
424
- cause: String(cause),
425
- }).pipe(Effect.as("error" as const)),
426
- ),
427
- );
435
+ const captureStatus = yield* captureWorkspaceCheckpoint({
436
+ cwd: input.scope.cwd,
437
+ checkpointRef,
438
+ ...(input.ordinalWithinScope > 0 ? { previousCheckpointRef } : {}),
439
+ }).pipe(
440
+ Effect.as("ready" as const),
441
+ Effect.catchTags({
442
+ VcsUnsupportedOperationError: () => Effect.succeed("missing" as const),
443
+ }),
444
+ Effect.catch((cause) =>
445
+ Effect.logWarning("orchestration V2 checkpoint capture failed", {
446
+ scopeId: input.scope.id,
447
+ checkpointRef,
448
+ cause: String(cause),
449
+ }).pipe(Effect.as("error" as const)),
450
+ ),
451
+ );
452
453
if (captureStatus !== "ready") {
454
return makeCheckpoint({