The best way to control Claude Code, Codex and more from any computer or phone
fix(server): new turns wait for overlapping checkpoint file restores (#23671)
Julius Marminge committed
Nov 12, 2026 at 20:31 UTC
0c4b1d8f277db542ac0142699748e42b80540814
5 files changed
+343
-9
apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts
+131
-1
@@ -32,7 +32,7 @@ import * as SqlitePersistence from "../persistence/Sqlite.ts";
32
import * as ServerConfig from "../config.ts";
33
import * as ServerSettings from "../serverSettings.ts";
34
import * as GitWorkflow from "../git/GitWorkflowService.ts";
35
-import { withWorkspaceLease } from "../workspace/workspaceLease.ts";
35
+import { withTurnStartReservation, withWorkspaceLease } from "../workspace/workspaceLease.ts";
36
import { ProviderWorkspaceMissingError } from "../provider/Errors.ts";
37
import { resolveCodexRollbackTurnCount } from "./Adapters/CodexAdapterV2.ts";
38
import * as CheckpointService from "./CheckpointService.ts";
@@ -394,6 +394,136 @@ it.effect.each([
394
}).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
395
);
396
397
+it.effect.each([{ restoreFiles: true }, { restoreFiles: false }])(
398
+ "holds turn starts inside the worktree during rollback with restoreFiles $restoreFiles",
399
+ ({ restoreFiles }) =>
400
+ Effect.gen(function* () {
401
+ const fileSystem = yield* FileSystem.FileSystem;
402
+ const path = yield* Path.Path;
403
+ const workspaceRoot = yield* fileSystem.realPath(
404
+ yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-rollback-reservation-" }),
405
+ );
406
+ const worktreesDir = path.join(workspaceRoot, "worktrees");
407
+ const worktreePath = path.join(worktreesDir, "repo", "feature");
408
+ yield* fileSystem.makeDirectory(path.join(worktreePath, ".git"), { recursive: true });
409
+ const threadId = ThreadId.make("rollback-reservation");
410
+ const providerThreadId = ProviderThreadId.make("rollback-reservation-provider-thread");
411
+ const providerInstanceId = ProviderInstanceId.make("rollback-reservation-provider");
412
+ const checkpointId = CheckpointId.make("rollback-reservation-checkpoint");
413
+ const scopeId = CheckpointScopeId.make("rollback-reservation-scope");
414
+ const release = Deferred.makeUnsafe<void>();
415
+ const openEntered = Deferred.makeUnsafe<void>();
416
+ const restoreEntered = Deferred.makeUnsafe<void>();
417
+ const releaseRestore = Deferred.makeUnsafe<void>();
418
+ const calls: string[] = [];
419
+ const projection = {
420
+ thread: {
421
+ id: threadId,
422
+ projectId: ProjectId.make("rollback-reservation-project"),
423
+ worktreePath,
424
+ branch: "feature/rollback-reservation",
425
+ activeProviderThreadId: providerThreadId,
426
+ modelSelection: { instanceId: providerInstanceId, model: "test-model" },
427
+ },
428
+ providerThread: {
429
+ id: providerThreadId,
430
+ providerSessionId: ProviderSessionId.make("rollback-reservation-session"),
431
+ providerInstanceId,
432
+ },
433
+ providerSession: undefined,
434
+ checkpoint: { id: checkpointId, scopeId, status: "ready", appRunOrdinal: 0 },
435
+ scope: { id: scopeId, cwd: worktreePath },
436
+ runsToRollback: [],
437
+ targetAttempt: undefined,
438
+ rolledBackAttemptIds: [],
439
+ staleCheckpoints: [],
440
+ rootNodes: [],
441
+ providerTurns: [],
442
+ } as unknown as ProjectionStore.ProjectionCheckpointRollbackContext;
443
+ const testLayer = CheckpointRollbackService.layer.pipe(
444
+ Layer.provide(
445
+ Layer.mergeAll(
446
+ ThreadCommandExecutor.layer,
447
+ NodeServices.layer,
448
+ Layer.mock(ServerConfig.ServerConfig)({ worktreesDir } as never),
449
+ ServerSettings.layerTest(),
450
+ Layer.mock(ProjectStore.ProjectStoreV2)({
451
+ get: () => Effect.succeedSome({ workspaceRoot } as never),
452
+ }),
453
+ Layer.mock(GitWorkflow.GitWorkflowService)({}),
454
+ Layer.mock(CheckpointService.CheckpointServiceV2)({
455
+ restore: () =>
456
+ Deferred.succeed(restoreEntered, undefined).pipe(
457
+ Effect.andThen(Deferred.await(releaseRestore)),
458
+ Effect.andThen(Effect.sync(() => calls.push("files"))),
459
+ ),
460
+ }),
461
+ Layer.mock(EventSink.EventSinkV2)({ write: () => Effect.succeed([]) }),
462
+ IdAllocator.layer,
463
+ Layer.mock(ProjectionStore.ProjectionStoreV2)({
464
+ getThread: () => Effect.succeed(projection.thread),
465
+ getCheckpointRollbackContext: () => Effect.succeed(projection),
466
+ getShellSnapshot: () =>
467
+ Effect.succeed({
468
+ schemaVersion: 1,
469
+ snapshotSequence: 0,
470
+ threads: [],
471
+ archivedThreads: [],
472
+ }),
473
+ }),
474
+ Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({
475
+ open: () =>
476
+ Deferred.succeed(openEntered, undefined).pipe(
477
+ Effect.andThen(Deferred.await(release)),
478
+ Effect.as({} as never),
479
+ ),
480
+ }),
481
+ Layer.mock(RuntimePolicy.RuntimePolicyV2)({
482
+ resolve: () => Effect.succeed({ cwd: worktreePath } as never),
483
+ }),
484
+ ),
485
+ ),
486
+ );
487
+ yield* Effect.gen(function* () {
488
+ const service = yield* CheckpointRollbackService.CheckpointRollbackServiceV2;
489
+ const rollback = yield* service
490
+ .execute({ threadId, providerThreadId, checkpointId, scopeId, restoreFiles })
491
+ .pipe(Effect.forkChild({ startImmediately: true }));
492
+ yield* Deferred.await(openEntered);
493
+ // A synchronous realPath lets the forked start reach the reservation
494
+ // before the next assertion, so "not entered" means "waiting".
495
+ const startTurn = withTurnStartReservation(
496
+ FileSystem.makeNoop({ realPath: (target) => Effect.succeed(target) }),
497
+ path,
498
+ path.join(worktreePath, "packages", "app"),
499
+ Effect.sync(() => calls.push("turn start")),
500
+ ).pipe(Effect.forkChild({ startImmediately: true }));
501
+ const turnStart = yield* startTurn;
502
+ if (restoreFiles) {
503
+ assert.deepEqual(calls, []);
504
+ yield* Deferred.succeed(release, undefined);
505
+ // The reservation must still be held while files are being restored,
506
+ // not only while the provider rollback is in flight.
507
+ yield* Deferred.await(restoreEntered);
508
+ const duringRestore = yield* startTurn;
509
+ assert.deepEqual(calls, []);
510
+ yield* Deferred.succeed(releaseRestore, undefined);
511
+ yield* Fiber.join(duringRestore);
512
+ } else {
513
+ // Nothing is rewritten, so the turn start must not wait for the provider.
514
+ yield* Fiber.join(turnStart);
515
+ yield* Deferred.succeed(release, undefined);
516
+ }
517
+ yield* Fiber.join(rollback);
518
+ yield* Fiber.join(turnStart);
519
+ assert.deepEqual(
520
+ calls,
521
+ restoreFiles ? ["files", "turn start", "turn start"] : ["turn start"],
522
+ );
523
+ }).pipe(Effect.provide(testLayer));
524
+ }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
525
+);
526
+
527
const seedRollbackHistory = Effect.gen(function* () {
528
const store = yield* ProjectionStore.ProjectionStoreV2;
529
const now = yield* DateTime.now;
apps/server/src/orchestration-v2/CheckpointRollbackService.ts
+13
-4
@@ -21,7 +21,10 @@ import * as GitWorkflowService from "../git/GitWorkflowService.ts";
21
import type * as ServerConfig from "../config.ts";
22
import type * as ServerSettings from "../serverSettings.ts";
23
import { makeIsManagedWorktreePath } from "../worktreesDirectory.ts";
24
-import { withWorkspaceLease } from "../workspace/workspaceLease.ts";
24
+import {
25
+ withCheckpointRestoreReservation,
26
+ withWorkspaceLease,
27
+} from "../workspace/workspaceLease.ts";
28
import {
29
isCheckpointRestoreIsolated,
30
SHARED_WORKSPACE_RESTORE_MESSAGE,
@@ -422,9 +425,15 @@ export const layer: Layer.Layer<
425
function* (input) {
426
// Storage cleanup removes worktrees under the same lease.
427
const thread = yield* projections.getThread(input.threadId);
425
- yield* thread.worktreePath === null
426
- ? execute(input)
427
- : withWorkspaceLease(path.resolve(thread.worktreePath), execute(input));
428
+ if (thread.worktreePath === null) return yield* execute(input);
429
+ const worktreePath = path.resolve(thread.worktreePath);
430
+ const rollback = withWorkspaceLease(worktreePath, execute(input));
431
+ // Shared-workspace restores are rejected and conversation-only rewinds
432
+ // touch no files; reserving for them would stall turn starts on the
433
+ // slow provider rollback for nothing.
434
+ yield* input.restoreFiles === false
435
+ ? rollback
436
+ : withCheckpointRestoreReservation(fileSystem, path, worktreePath, rollback);
437
},
438
(effect, input) =>
439
effect.pipe(
apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts
+90
-1
@@ -35,7 +35,10 @@ import * as PlatformError from "effect/PlatformError";
35
import * as Schema from "effect/Schema";
36
37
import * as ServerConfig from "../config.ts";
38
-import { withWorkspaceLease } from "../workspace/workspaceLease.ts";
38
+import {
39
+ withCheckpointRestoreReservation,
40
+ withWorkspaceLease,
41
+} from "../workspace/workspaceLease.ts";
42
import * as GitWorkflow from "../git/GitWorkflowService.ts";
43
import * as GitVcsDriver from "../vcs/GitVcsDriver.ts";
44
import { layerRealGit } from "./ProviderTurnStartService.testkit.ts";
@@ -1565,6 +1568,92 @@ effectIt.effect.each(["cleanup-first", "startup-first"])(
1568
}).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
1569
);
1570
1571
+// regression: #12341 a turn start overlapping a checkpoint restore wrote into the checkout
1572
+// The filesystem is synchronous so the turn start either reaches the provider or parks on the
1573
+// lease before the restore is released; realPath models the symlink alias and missing worktrees.
1574
+effectIt.effect.each([
1575
+ {
1576
+ overlap: "a project-root thread during a restore of the project checkout",
1577
+ restorePath: "/tmp/native-account-project",
1578
+ worktreePath: undefined,
1579
+ realPaths: {},
1580
+ missing: [],
1581
+ },
1582
+ {
1583
+ overlap: "a symlink alias of the restoring checkout",
1584
+ restorePath: "/tmp/restore-overlap/link",
1585
+ worktreePath: "/tmp/restore-overlap/checkout",
1586
+ realPaths: { "/tmp/restore-overlap/link": "/tmp/restore-overlap/checkout" },
1587
+ missing: [],
1588
+ },
1589
+ {
1590
+ overlap: "a directory inside the restoring checkout",
1591
+ restorePath: "/tmp/restore-overlap/checkout",
1592
+ worktreePath: "/tmp/restore-overlap/checkout/packages/app",
1593
+ realPaths: {},
1594
+ missing: [],
1595
+ },
1596
+ {
1597
+ overlap: "a not-yet-recreated worktree under a symlink alias of the restoring one",
1598
+ restorePath: "/tmp/restore-overlap/link/missing-child",
1599
+ worktreePath: "/tmp/restore-overlap/checkout/missing-child",
1600
+ realPaths: { "/tmp/restore-overlap/link": "/tmp/restore-overlap/checkout" },
1601
+ missing: [
1602
+ "/tmp/restore-overlap/link/missing-child",
1603
+ "/tmp/restore-overlap/checkout/missing-child",
1604
+ ],
1605
+ },
1606
+] as const)(
1607
+ "does not start provider work in $overlap until the restore finishes",
1608
+ ({ restorePath, worktreePath, realPaths, missing }) =>
1609
+ Effect.gen(function* () {
1610
+ const order: Array<string> = [];
1611
+ const releaseRestore = yield* Deferred.make<void>();
1612
+ const restoreEntered = yield* Deferred.make<void>();
1613
+ const fileSystem = FileSystem.makeNoop({
1614
+ exists: () => Effect.succeed(true),
1615
+ realPath: (candidate) =>
1616
+ (missing as ReadonlyArray<string>).includes(candidate)
1617
+ ? Effect.fail(
1618
+ PlatformError.systemError({
1619
+ _tag: "NotFound",
1620
+ module: "FileSystem",
1621
+ method: "realPath",
1622
+ pathOrDescriptor: candidate,
1623
+ }),
1624
+ )
1625
+ : Effect.succeed((realPaths as Record<string, string>)[candidate] ?? candidate),
1626
+ });
1627
+ const harness = makeLocalCommandHarness({
1628
+ text: "Continue",
1629
+ ...(worktreePath === undefined ? {} : { worktreePath }),
1630
+ fileSystem,
1631
+ failReadsAfterRunning: true,
1632
+ });
1633
+ const open = harness.open.getMockImplementation()!;
1634
+ harness.open.mockImplementation(() =>
1635
+ Effect.sync(() => order.push("provider session opened")).pipe(Effect.andThen(open())),
1636
+ );
1637
+ // The reservation CheckpointRollbackServiceV2.execute holds around its
1638
+ // isolation check, provider rollback, and git restore of `restorePath`.
1639
+ const restore = yield* withCheckpointRestoreReservation(
1640
+ fileSystem,
1641
+ yield* Path.Path,
1642
+ restorePath,
1643
+ Deferred.succeed(restoreEntered, undefined).pipe(
1644
+ Effect.andThen(Deferred.await(releaseRestore)),
1645
+ Effect.andThen(Effect.sync(() => order.push("checkpoint files restored"))),
1646
+ ),
1647
+ ).pipe(Effect.forkChild({ startImmediately: true }));
1648
+ yield* Deferred.await(restoreEntered);
1649
+ const startup = yield* harness.start.pipe(Effect.forkChild({ startImmediately: true }));
1650
+ yield* Deferred.succeed(releaseRestore, undefined);
1651
+ yield* Fiber.join(restore);
1652
+ yield* Fiber.join(startup);
1653
+ expect(order).toEqual(["checkpoint files restored", "provider session opened"]);
1654
+ }).pipe(Effect.provide(Path.layer)),
1655
+);
1656
+
1657
effectIt.effect("reads current startup state after waiting for the workspace lease", () =>
1658
Effect.gen(function* () {
1659
const worktreePath = "/tmp/provider-start-fresh-state";
apps/server/src/orchestration-v2/ProviderTurnStartService.ts
+17
-3
@@ -30,7 +30,7 @@ import {
30
CLAUDE_PROVIDER,
31
CLAUDE_QUERY_ENDED_BEFORE_TURN_DETAIL,
32
} from "./Adapters/ClaudeAdapterV2.ts";
33
-import { withWorkspaceLease } from "../workspace/workspaceLease.ts";
33
+import { withTurnStartReservation, withWorkspaceLease } from "../workspace/workspaceLease.ts";
34
import * as GitWorkflowService from "../git/GitWorkflowService.ts";
35
import * as ProjectService from "../project/ProjectService.ts";
36
import * as ProviderAuthService from "../provider/ProviderAuthService.ts";
@@ -154,7 +154,9 @@ export const layer: Layer.Layer<
154
const turnAnalytics = yield* ProviderUsageAnalytics.ProviderTurnAnalytics;
155
const contextHandoffService = yield* ContextHandoffService.ContextHandoffServiceV2;
156
const idAllocator = yield* IdAllocator.IdAllocatorV2;
157
+ const fileSystem = yield* FileSystem.FileSystem;
158
const path = yield* Path.Path;
159
+ const projects = yield* ProjectService.ProjectService;
160
const providerAuth = yield* ProviderAuthService.ProviderAuthService;
161
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
162
const providerSessions = yield* ProviderSessionManager.ProviderSessionManagerV2;
@@ -1763,9 +1765,21 @@ export const layer: Layer.Layer<
1765
// Storage cleanup removes worktrees under the same lease. Only startup
1766
// holds it; the provider turn runs after it is released.
1767
const { thread } = yield* projectionStore.getThreadRecords(input.threadId, []);
1766
- const providerTurn = yield* thread.worktreePath === null
1768
+ const cwd =
1769
+ thread.worktreePath ??
1770
+ (yield* projects
1771
+ .getById(thread.projectId, { includeDeleted: true })
1772
+ .pipe(Effect.map((project) => Option.getOrUndefined(project)?.workspaceRoot)));
1773
+ const providerTurn = yield* cwd === undefined
1774
? start(input)
1768
- : withWorkspaceLease(path.resolve(thread.worktreePath), start(input));
1775
+ : withTurnStartReservation(
1776
+ fileSystem,
1777
+ path,
1778
+ path.resolve(cwd),
1779
+ thread.worktreePath === null
1780
+ ? start(input)
1781
+ : withWorkspaceLease(path.resolve(thread.worktreePath), start(input)),
1782
+ );
1783
if (providerTurn !== undefined) yield* providerTurn;
1784
},
1785
(effect, input) =>
apps/server/src/workspace/workspaceLease.ts
+92
@@ -1,4 +1,7 @@
1
+import * as Deferred from "effect/Deferred";
2
import * as Effect from "effect/Effect";
3
+import type * as FileSystem from "effect/FileSystem";
4
+import type * as Path from "effect/Path";
5
import * as Semaphore from "effect/Semaphore";
6
7
const leases = new Map<string, { semaphore: Semaphore.Semaphore; users: number }>();
@@ -20,3 +23,92 @@ export const withWorkspaceLease = <A, E, R>(
23
),
24
);
25
});
26
+
27
+// A restore rewrites files under its directory, so it must exclude provider
28
+// startup in any overlapping workspace: symlink aliases, parents, and children.
29
+// Kept apart from withWorkspaceLease because that lease is non-reentrant and
30
+// restores call Git work that leases the project root.
31
+interface Reservation {
32
+ readonly cwd: string;
33
+ readonly kind: "restore" | "turn-start";
34
+}
35
+
36
+const reservations = new Set<Reservation>();
37
+let reservationReleased = Deferred.makeUnsafe<void>();
38
+
39
+// path.relative folds win32 drive-letter case and treats a POSIX backslash as
40
+// part of a name, unlike a raw prefix check.
41
+const contains = (path: Path.Path, parent: string, child: string) => {
42
+ const relative = path.relative(parent, child);
43
+ return (
44
+ relative === "" ||
45
+ (!path.isAbsolute(relative) && relative !== ".." && !relative.startsWith(`..${path.sep}`))
46
+ );
47
+};
48
+
49
+const conflicts = (path: Path.Path, a: Reservation, b: Reservation) =>
50
+ (a.kind === "restore" || b.kind === "restore") &&
51
+ (contains(path, a.cwd, b.cwd) || contains(path, b.cwd, a.cwd));
52
+
53
+const reserve = (path: Path.Path, reservation: Reservation): Effect.Effect<void> =>
54
+ Effect.suspend(() => {
55
+ for (const held of reservations) {
56
+ if (conflicts(path, held, reservation)) {
57
+ return Deferred.await(reservationReleased).pipe(Effect.andThen(reserve(path, reservation)));
58
+ }
59
+ }
60
+ reservations.add(reservation);
61
+ return Effect.void;
62
+ });
63
+
64
+const release = (reservation: Reservation) =>
65
+ Effect.sync(() => {
66
+ if (!reservations.delete(reservation)) return;
67
+ const released = reservationReleased;
68
+ reservationReleased = Deferred.makeUnsafe<void>();
69
+ Deferred.doneUnsafe(released, Effect.void);
70
+ });
71
+
72
+// A worktree removed by storage cleanup is recreated inside the reserved
73
+// section, so resolve its nearest existing ancestor: a symlink alias above it
74
+// must still canonicalize the same on both sides. Walks at most the path depth.
75
+const canonicalize = (
76
+ fileSystem: FileSystem.FileSystem,
77
+ path: Path.Path,
78
+ target: string,
79
+): Effect.Effect<string> =>
80
+ fileSystem.realPath(target).pipe(
81
+ Effect.catchReason("PlatformError", "NotFound", () => {
82
+ const parent = path.dirname(target);
83
+ return parent === target
84
+ ? Effect.succeed(target)
85
+ : canonicalize(fileSystem, path, parent).pipe(
86
+ Effect.map((resolved) => path.join(resolved, path.basename(target))),
87
+ );
88
+ }),
89
+ Effect.orElseSucceed(() => target),
90
+ );
91
+
92
+const withReservation =
93
+ (kind: Reservation["kind"]) =>
94
+ <A, E, R>(
95
+ fileSystem: FileSystem.FileSystem,
96
+ path: Path.Path,
97
+ cwd: string,
98
+ effect: Effect.Effect<A, E, R>,
99
+ ): Effect.Effect<A, E, R> =>
100
+ canonicalize(fileSystem, path, cwd).pipe(
101
+ Effect.flatMap((canonical) => {
102
+ const reservation: Reservation = { cwd: canonical, kind };
103
+ return reserve(path, reservation).pipe(
104
+ Effect.andThen(effect),
105
+ Effect.ensuring(release(reservation)),
106
+ );
107
+ }),
108
+ );
109
+
110
+/** Exclusive over every overlapping workspace while checkpoint files are restored. */
111
+export const withCheckpointRestoreReservation = withReservation("restore");
112
+
113
+/** Waits out overlapping restores; turn startups never wait on each other. */
114
+export const withTurnStartReservation = withReservation("turn-start");