@devries-cameron20 / t3code Mirror · read-only / commits / 61085457d7
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

fix(server): repeated capture timeouts no longer delay turn starts (#29196)

Co-authored-by: maria-rcks <maria@kuuro.net>

Lakshmi Tanmay committed Nov 20, 2026 at 01:32 UTC 61085457d76ea246b5955c092ffba2452a825aff
2 files changed +179 -33
apps/server/src/orchestration-v2/CheckpointService.test.ts
+126 -4
@@ -11,6 +11,7 @@ import {
11 VcsProcessTimeoutError,
12 VcsUnsupportedOperationError,
13 } from "@t3tools/contracts";
14 +import * as Cause from "effect/Cause";
15 import * as DateTime from "effect/DateTime";
16 import * as Deferred from "effect/Deferred";
17 import * as Effect from "effect/Effect";
@@ -164,12 +165,23 @@ it.effect(
165 assert.strictEqual((yield* Fiber.join(postTurn)).status, "ready");
166 assert.strictEqual((yield* Fiber.join(waitingBaseline))._tag, "Success");
167
167 - const slowBaseline = yield* checkpoints
168 - .captureBaseline({ scope, ordinalWithinScope: 2 })
168 + for (const ordinalWithinScope of [2, 3]) {
169 + const slowBaseline = yield* checkpoints
170 + .captureBaseline({ scope, ordinalWithinScope })
171 + .pipe(Effect.result, Effect.forkChild);
172 + yield* TestClock.adjust("5 seconds");
173 + const result = yield* Fiber.join(slowBaseline);
174 + assert.strictEqual(result._tag, "Failure");
175 + if (result._tag === "Failure") {
176 + assert.isTrue(Cause.isTimeoutError(result.failure.cause));
177 + }
178 + }
179 + const backedOffBaseline = yield* checkpoints
180 + .captureBaseline({ scope, ordinalWithinScope: 4 })
181 .pipe(Effect.result, Effect.forkChild);
182 yield* TestClock.adjust("5 seconds");
171 - assert.strictEqual((yield* Fiber.join(slowBaseline))._tag, "Failure");
172 - assert.strictEqual(yield* Ref.get(snapshots), 2);
183 + assert.strictEqual((yield* Fiber.join(backedOffBaseline))._tag, "Success");
184 + assert.strictEqual(yield* Ref.get(snapshots), 3);
185 assert.strictEqual((yield* Ref.get(refs)).size, 1);
186 }).pipe(Effect.provide(layerTest));
187 }),
@@ -233,3 +245,113 @@ it.effect("records a repository probe failure as an error checkpoint, not a miss
245 assert.deepEqual(statuses, { baseline: "error", turn: "error" });
246 }),
247 );
248 +
249 +it.effect("backs off only baseline retries after repeated workspace capture timeouts", () =>
250 + Effect.gen(function* () {
251 + const scope: OrchestrationV2CheckpointScope = {
252 + id: CheckpointScopeId.make("checkpoint-scope:slow-capture"),
253 + threadId: ThreadId.make("thread:slow-capture"),
254 + runId: RunId.make("run:slow-capture:1"),
255 + nodeId: NodeId.make("node:slow-capture:1"),
256 + parentScopeId: null,
257 + providerThreadId: ProviderThreadId.make("provider-thread:slow-capture"),
258 + kind: "root_run",
259 + ordinalWithinParent: 0,
260 + advancesAppRunCount: true,
261 + cwd: "/repo/slow",
262 + createdAt: DateTime.makeUnsafe("2026-07-28T00:00:00.000Z"),
263 + };
264 + const timeout = new VcsProcessTimeoutError({
265 + operation: "GitVcsDriver.checkpoints.captureCheckpoint",
266 + command: "git",
267 + cwd: scope.cwd,
268 + timeoutMs: 30_000,
269 + });
270 + let attempts = 0;
271 + let healthy = false;
272 + let nonTimeoutFailure = false;
273 + const failure = new VcsProcessExitError({
274 + operation: "test.capture",
275 + command: "git",
276 + cwd: scope.cwd,
277 + exitCode: 128,
278 + detail: "capture failed",
279 + });
280 + const layerTest = CheckpointService.layer.pipe(
281 + Layer.provide(
282 + Layer.mergeAll(
283 + IdAllocator.layer,
284 + Layer.mock(CheckpointStore.CheckpointStore)({
285 + isGitRepository: () => Effect.succeed(true),
286 + hasCheckpointRef: () => Effect.succeed(false),
287 + captureCheckpoint: () =>
288 + Effect.suspend(() => {
289 + attempts++;
290 + return healthy ? Effect.void : Effect.fail(nonTimeoutFailure ? failure : timeout);
291 + }),
292 + }),
293 + ),
294 + ),
295 + Layer.provideMerge(NodeCrypto.layer),
296 + );
297 + yield* Effect.gen(function* () {
298 + const checkpoints = yield* CheckpointService.CheckpointServiceV2;
299 + const baseline = { scope, ordinalWithinScope: 0 };
300 + const turn = {
301 + scope,
302 + ordinalWithinScope: 1,
303 + runId: scope.runId,
304 + nodeId: scope.nodeId,
305 + appRunOrdinal: 1,
306 + capturedAt: scope.createdAt,
307 + };
308 + yield* Effect.result(checkpoints.captureBaseline(baseline));
309 + assert.strictEqual((yield* checkpoints.capture(turn)).status, "error");
310 + assert.strictEqual(attempts, 2);
311 +
312 + const otherThread = { ...scope, id: CheckpointScopeId.make("checkpoint-scope:other") };
313 + const skipped = yield* Effect.result(
314 + checkpoints.captureBaseline({ scope: otherThread, ordinalWithinScope: 0 }),
315 + );
316 + assert.strictEqual(attempts, 2, "baseline retried a repeatedly timing-out workspace");
317 + assert.strictEqual(skipped._tag, "Success");
318 +
319 + yield* Effect.result(
320 + checkpoints.captureBaseline({
321 + scope: { ...otherThread, cwd: "/repo/other" },
322 + ordinalWithinScope: 0,
323 + }),
324 + );
325 + assert.strictEqual(attempts, 3, "another workspace must still attempt capture");
326 +
327 + const failedTurn = yield* checkpoints.capture(turn);
328 + assert.strictEqual(failedTurn.status, "error");
329 + assert.strictEqual(attempts, 4, "post-turn capture must not back off");
330 +
331 + yield* checkpoints.captureBaseline(baseline);
332 + assert.strictEqual(attempts, 4, "baseline retries must wait for post-turn recovery");
333 + healthy = true;
334 + assert.strictEqual((yield* checkpoints.capture(turn)).status, "ready");
335 + assert.strictEqual(attempts, 5);
336 + yield* checkpoints.captureBaseline(baseline);
337 + assert.strictEqual(attempts, 6, "a successful post-turn capture must clear backoff");
338 +
339 + healthy = false;
340 + yield* Effect.result(checkpoints.captureBaseline(baseline));
341 + yield* Effect.result(checkpoints.captureBaseline(baseline));
342 + yield* checkpoints.captureBaseline(baseline);
343 + assert.strictEqual(attempts, 8);
344 + nonTimeoutFailure = true;
345 + assert.strictEqual((yield* checkpoints.capture(turn)).status, "error");
346 + nonTimeoutFailure = false;
347 + yield* Effect.result(checkpoints.captureBaseline(baseline));
348 + assert.strictEqual(attempts, 10, "a non-timeout failure must clear backoff");
349 + healthy = true;
350 + yield* checkpoints.captureBaseline(baseline);
351 + healthy = false;
352 + yield* Effect.result(checkpoints.captureBaseline(baseline));
353 + yield* Effect.result(checkpoints.captureBaseline(baseline));
354 + assert.strictEqual(attempts, 13, "a successful baseline must clear the timeout streak");
355 + }).pipe(Effect.provide(layerTest));
356 + }),
357 +);
apps/server/src/orchestration-v2/CheckpointService.ts
+53 -29
@@ -9,6 +9,7 @@ import {
9 RunId,
10 ThreadId,
11 } from "@t3tools/contracts";
12 +import * as Cache from "effect/Cache";
13 import * as Context from "effect/Context";
14 import * as Digest from "effect/crypto/Digest";
15 import * as DateTime from "effect/DateTime";
@@ -17,6 +18,7 @@ import * as Base64Url from "effect/encoding/Base64Url";
18 import * as Hex from "effect/encoding/Hex";
19 import * as KeyedLock from "@t3tools/shared/KeyedLock";
20 import * as Layer from "effect/Layer";
21 +import * as Option from "effect/Option";
22 import * as Schema from "effect/Schema";
23
24 import { isGitImport, parseTurnDiffFilesFromNumstat } from "../checkpointing/Diffs.ts";
@@ -250,6 +252,28 @@ export const layer: Layer.Layer<
252 const withWorkspaceLock = <A, E, R>(cwd: string, effect: Effect.Effect<A, E, R>) =>
253 workspaceLocks.withLock(cwd, effect);
254
255 + const captureTimeouts = yield* Cache.make<string, number>({
256 + capacity: 128,
257 + lookup: () => Effect.succeed(0),
258 + });
259 + // Both callers hold the workspace lock; post-turn captures always probe for recovery.
260 + const captureWorkspaceCheckpoint = Effect.fn("CheckpointService.captureWorkspaceCheckpoint")(
261 + function* (input: CheckpointStore.CaptureCheckpointInput, baseline = false) {
262 + const capture = checkpointStore.captureCheckpoint(input);
263 + yield* (baseline ? capture.pipe(Effect.timeout(BASELINE_CAPTURE_BUDGET)) : capture).pipe(
264 + Effect.tap(() => Cache.invalidate(captureTimeouts, input.cwd)),
265 + Effect.tapError((error) =>
266 + error._tag === "VcsProcessTimeoutError" || error._tag === "TimeoutError"
267 + ? Effect.gen(function* () {
268 + const previous = yield* Cache.get(captureTimeouts, input.cwd);
269 + yield* Cache.set(captureTimeouts, input.cwd, previous + 1);
270 + })
271 + : Cache.invalidate(captureTimeouts, input.cwd),
272 + ),
273 + );
274 + },
275 + );
276 +
277 // A failed probe is an operational error, not proof that the workspace has no repository.
278 const probeRepository = (scope: OrchestrationV2CheckpointScope) =>
279 checkpointStore.isGitRepository(scope.cwd).pipe(
@@ -284,17 +308,19 @@ export const layer: Layer.Layer<
308 return;
309 }
310
287 - yield* checkpointStore
288 - .captureCheckpoint({
289 - cwd: input.scope.cwd,
290 - checkpointRef,
291 - })
292 - .pipe(
293 - Effect.timeout(BASELINE_CAPTURE_BUDGET),
294 - Effect.catchTags({
295 - VcsUnsupportedOperationError: () => Effect.void,
296 - }),
311 + const timeouts = yield* Cache.getOption(captureTimeouts, input.scope.cwd);
312 + if (Option.isSome(timeouts) && timeouts.value >= 2) {
313 + return yield* Effect.logWarning(
314 + "orchestration V2 checkpoint baseline skipped after repeated capture timeouts",
315 + { scopeId: input.scope.id, ordinalWithinScope: input.ordinalWithinScope },
316 );
317 + }
318 +
319 + yield* captureWorkspaceCheckpoint({ cwd: input.scope.cwd, checkpointRef }, true).pipe(
320 + Effect.catchTags({
321 + VcsUnsupportedOperationError: () => Effect.void,
322 + }),
323 + );
324 }),
325 ).pipe(
326 Effect.mapError(
@@ -406,25 +432,23 @@ export const layer: Layer.Layer<
432 });
433 }
434
409 - const captureStatus = yield* checkpointStore
410 - .captureCheckpoint({
411 - cwd: input.scope.cwd,
412 - checkpointRef,
413 - ...(input.ordinalWithinScope > 0 ? { previousCheckpointRef } : {}),
414 - })
415 - .pipe(
416 - Effect.as("ready" as const),
417 - Effect.catchTags({
418 - VcsUnsupportedOperationError: () => Effect.succeed("missing" as const),
419 - }),
420 - Effect.catch((cause) =>
421 - Effect.logWarning("orchestration V2 checkpoint capture failed", {
422 - scopeId: input.scope.id,
423 - checkpointRef,
424 - cause: String(cause),
425 - }).pipe(Effect.as("error" as const)),
426 - ),
427 - );
435 + const captureStatus = yield* captureWorkspaceCheckpoint({
436 + cwd: input.scope.cwd,
437 + checkpointRef,
438 + ...(input.ordinalWithinScope > 0 ? { previousCheckpointRef } : {}),
439 + }).pipe(
440 + Effect.as("ready" as const),
441 + Effect.catchTags({
442 + VcsUnsupportedOperationError: () => Effect.succeed("missing" as const),
443 + }),
444 + Effect.catch((cause) =>
445 + Effect.logWarning("orchestration V2 checkpoint capture failed", {
446 + scopeId: input.scope.id,
447 + checkpointRef,
448 + cause: String(cause),
449 + }).pipe(Effect.as("error" as const)),
450 + ),
451 + );
452
453 if (captureStatus !== "ready") {
454 return makeCheckpoint({