@devries-cameron20 / t3code Mirror · read-only / commits / 0c4b1d8f27
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 17:15 UTC

fix(server): new turns wait for overlapping checkpoint file restores (#23671)

Julius Marminge committed Nov 12, 2026 at 20:31 UTC 0c4b1d8f277db542ac0142699748e42b80540814
5 files changed +343 -9
apps/server/src/orchestration-v2/CheckpointRollbackService.test.ts
+131 -1
@@ -32,7 +32,7 @@ import * as SqlitePersistence from "../persistence/Sqlite.ts";
32 import * as ServerConfig from "../config.ts";
33 import * as ServerSettings from "../serverSettings.ts";
34 import * as GitWorkflow from "../git/GitWorkflowService.ts";
35 -import { withWorkspaceLease } from "../workspace/workspaceLease.ts";
35 +import { withTurnStartReservation, withWorkspaceLease } from "../workspace/workspaceLease.ts";
36 import { ProviderWorkspaceMissingError } from "../provider/Errors.ts";
37 import { resolveCodexRollbackTurnCount } from "./Adapters/CodexAdapterV2.ts";
38 import * as CheckpointService from "./CheckpointService.ts";
@@ -394,6 +394,136 @@ it.effect.each([
394 }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
395 );
396
397 +it.effect.each([{ restoreFiles: true }, { restoreFiles: false }])(
398 + "holds turn starts inside the worktree during rollback with restoreFiles $restoreFiles",
399 + ({ restoreFiles }) =>
400 + Effect.gen(function* () {
401 + const fileSystem = yield* FileSystem.FileSystem;
402 + const path = yield* Path.Path;
403 + const workspaceRoot = yield* fileSystem.realPath(
404 + yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-rollback-reservation-" }),
405 + );
406 + const worktreesDir = path.join(workspaceRoot, "worktrees");
407 + const worktreePath = path.join(worktreesDir, "repo", "feature");
408 + yield* fileSystem.makeDirectory(path.join(worktreePath, ".git"), { recursive: true });
409 + const threadId = ThreadId.make("rollback-reservation");
410 + const providerThreadId = ProviderThreadId.make("rollback-reservation-provider-thread");
411 + const providerInstanceId = ProviderInstanceId.make("rollback-reservation-provider");
412 + const checkpointId = CheckpointId.make("rollback-reservation-checkpoint");
413 + const scopeId = CheckpointScopeId.make("rollback-reservation-scope");
414 + const release = Deferred.makeUnsafe<void>();
415 + const openEntered = Deferred.makeUnsafe<void>();
416 + const restoreEntered = Deferred.makeUnsafe<void>();
417 + const releaseRestore = Deferred.makeUnsafe<void>();
418 + const calls: string[] = [];
419 + const projection = {
420 + thread: {
421 + id: threadId,
422 + projectId: ProjectId.make("rollback-reservation-project"),
423 + worktreePath,
424 + branch: "feature/rollback-reservation",
425 + activeProviderThreadId: providerThreadId,
426 + modelSelection: { instanceId: providerInstanceId, model: "test-model" },
427 + },
428 + providerThread: {
429 + id: providerThreadId,
430 + providerSessionId: ProviderSessionId.make("rollback-reservation-session"),
431 + providerInstanceId,
432 + },
433 + providerSession: undefined,
434 + checkpoint: { id: checkpointId, scopeId, status: "ready", appRunOrdinal: 0 },
435 + scope: { id: scopeId, cwd: worktreePath },
436 + runsToRollback: [],
437 + targetAttempt: undefined,
438 + rolledBackAttemptIds: [],
439 + staleCheckpoints: [],
440 + rootNodes: [],
441 + providerTurns: [],
442 + } as unknown as ProjectionStore.ProjectionCheckpointRollbackContext;
443 + const testLayer = CheckpointRollbackService.layer.pipe(
444 + Layer.provide(
445 + Layer.mergeAll(
446 + ThreadCommandExecutor.layer,
447 + NodeServices.layer,
448 + Layer.mock(ServerConfig.ServerConfig)({ worktreesDir } as never),
449 + ServerSettings.layerTest(),
450 + Layer.mock(ProjectStore.ProjectStoreV2)({
451 + get: () => Effect.succeedSome({ workspaceRoot } as never),
452 + }),
453 + Layer.mock(GitWorkflow.GitWorkflowService)({}),
454 + Layer.mock(CheckpointService.CheckpointServiceV2)({
455 + restore: () =>
456 + Deferred.succeed(restoreEntered, undefined).pipe(
457 + Effect.andThen(Deferred.await(releaseRestore)),
458 + Effect.andThen(Effect.sync(() => calls.push("files"))),
459 + ),
460 + }),
461 + Layer.mock(EventSink.EventSinkV2)({ write: () => Effect.succeed([]) }),
462 + IdAllocator.layer,
463 + Layer.mock(ProjectionStore.ProjectionStoreV2)({
464 + getThread: () => Effect.succeed(projection.thread),
465 + getCheckpointRollbackContext: () => Effect.succeed(projection),
466 + getShellSnapshot: () =>
467 + Effect.succeed({
468 + schemaVersion: 1,
469 + snapshotSequence: 0,
470 + threads: [],
471 + archivedThreads: [],
472 + }),
473 + }),
474 + Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({
475 + open: () =>
476 + Deferred.succeed(openEntered, undefined).pipe(
477 + Effect.andThen(Deferred.await(release)),
478 + Effect.as({} as never),
479 + ),
480 + }),
481 + Layer.mock(RuntimePolicy.RuntimePolicyV2)({
482 + resolve: () => Effect.succeed({ cwd: worktreePath } as never),
483 + }),
484 + ),
485 + ),
486 + );
487 + yield* Effect.gen(function* () {
488 + const service = yield* CheckpointRollbackService.CheckpointRollbackServiceV2;
489 + const rollback = yield* service
490 + .execute({ threadId, providerThreadId, checkpointId, scopeId, restoreFiles })
491 + .pipe(Effect.forkChild({ startImmediately: true }));
492 + yield* Deferred.await(openEntered);
493 + // A synchronous realPath lets the forked start reach the reservation
494 + // before the next assertion, so "not entered" means "waiting".
495 + const startTurn = withTurnStartReservation(
496 + FileSystem.makeNoop({ realPath: (target) => Effect.succeed(target) }),
497 + path,
498 + path.join(worktreePath, "packages", "app"),
499 + Effect.sync(() => calls.push("turn start")),
500 + ).pipe(Effect.forkChild({ startImmediately: true }));
501 + const turnStart = yield* startTurn;
502 + if (restoreFiles) {
503 + assert.deepEqual(calls, []);
504 + yield* Deferred.succeed(release, undefined);
505 + // The reservation must still be held while files are being restored,
506 + // not only while the provider rollback is in flight.
507 + yield* Deferred.await(restoreEntered);
508 + const duringRestore = yield* startTurn;
509 + assert.deepEqual(calls, []);
510 + yield* Deferred.succeed(releaseRestore, undefined);
511 + yield* Fiber.join(duringRestore);
512 + } else {
513 + // Nothing is rewritten, so the turn start must not wait for the provider.
514 + yield* Fiber.join(turnStart);
515 + yield* Deferred.succeed(release, undefined);
516 + }
517 + yield* Fiber.join(rollback);
518 + yield* Fiber.join(turnStart);
519 + assert.deepEqual(
520 + calls,
521 + restoreFiles ? ["files", "turn start", "turn start"] : ["turn start"],
522 + );
523 + }).pipe(Effect.provide(testLayer));
524 + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
525 +);
526 +
527 const seedRollbackHistory = Effect.gen(function* () {
528 const store = yield* ProjectionStore.ProjectionStoreV2;
529 const now = yield* DateTime.now;
apps/server/src/orchestration-v2/CheckpointRollbackService.ts
+13 -4
@@ -21,7 +21,10 @@ import * as GitWorkflowService from "../git/GitWorkflowService.ts";
21 import type * as ServerConfig from "../config.ts";
22 import type * as ServerSettings from "../serverSettings.ts";
23 import { makeIsManagedWorktreePath } from "../worktreesDirectory.ts";
24 -import { withWorkspaceLease } from "../workspace/workspaceLease.ts";
24 +import {
25 + withCheckpointRestoreReservation,
26 + withWorkspaceLease,
27 +} from "../workspace/workspaceLease.ts";
28 import {
29 isCheckpointRestoreIsolated,
30 SHARED_WORKSPACE_RESTORE_MESSAGE,
@@ -422,9 +425,15 @@ export const layer: Layer.Layer<
425 function* (input) {
426 // Storage cleanup removes worktrees under the same lease.
427 const thread = yield* projections.getThread(input.threadId);
425 - yield* thread.worktreePath === null
426 - ? execute(input)
427 - : withWorkspaceLease(path.resolve(thread.worktreePath), execute(input));
428 + if (thread.worktreePath === null) return yield* execute(input);
429 + const worktreePath = path.resolve(thread.worktreePath);
430 + const rollback = withWorkspaceLease(worktreePath, execute(input));
431 + // Shared-workspace restores are rejected and conversation-only rewinds
432 + // touch no files; reserving for them would stall turn starts on the
433 + // slow provider rollback for nothing.
434 + yield* input.restoreFiles === false
435 + ? rollback
436 + : withCheckpointRestoreReservation(fileSystem, path, worktreePath, rollback);
437 },
438 (effect, input) =>
439 effect.pipe(
apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts
+90 -1
@@ -35,7 +35,10 @@ import * as PlatformError from "effect/PlatformError";
35 import * as Schema from "effect/Schema";
36
37 import * as ServerConfig from "../config.ts";
38 -import { withWorkspaceLease } from "../workspace/workspaceLease.ts";
38 +import {
39 + withCheckpointRestoreReservation,
40 + withWorkspaceLease,
41 +} from "../workspace/workspaceLease.ts";
42 import * as GitWorkflow from "../git/GitWorkflowService.ts";
43 import * as GitVcsDriver from "../vcs/GitVcsDriver.ts";
44 import { layerRealGit } from "./ProviderTurnStartService.testkit.ts";
@@ -1565,6 +1568,92 @@ effectIt.effect.each(["cleanup-first", "startup-first"])(
1568 }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
1569 );
1570
1571 +// regression: #12341 a turn start overlapping a checkpoint restore wrote into the checkout
1572 +// The filesystem is synchronous so the turn start either reaches the provider or parks on the
1573 +// lease before the restore is released; realPath models the symlink alias and missing worktrees.
1574 +effectIt.effect.each([
1575 + {
1576 + overlap: "a project-root thread during a restore of the project checkout",
1577 + restorePath: "/tmp/native-account-project",
1578 + worktreePath: undefined,
1579 + realPaths: {},
1580 + missing: [],
1581 + },
1582 + {
1583 + overlap: "a symlink alias of the restoring checkout",
1584 + restorePath: "/tmp/restore-overlap/link",
1585 + worktreePath: "/tmp/restore-overlap/checkout",
1586 + realPaths: { "/tmp/restore-overlap/link": "/tmp/restore-overlap/checkout" },
1587 + missing: [],
1588 + },
1589 + {
1590 + overlap: "a directory inside the restoring checkout",
1591 + restorePath: "/tmp/restore-overlap/checkout",
1592 + worktreePath: "/tmp/restore-overlap/checkout/packages/app",
1593 + realPaths: {},
1594 + missing: [],
1595 + },
1596 + {
1597 + overlap: "a not-yet-recreated worktree under a symlink alias of the restoring one",
1598 + restorePath: "/tmp/restore-overlap/link/missing-child",
1599 + worktreePath: "/tmp/restore-overlap/checkout/missing-child",
1600 + realPaths: { "/tmp/restore-overlap/link": "/tmp/restore-overlap/checkout" },
1601 + missing: [
1602 + "/tmp/restore-overlap/link/missing-child",
1603 + "/tmp/restore-overlap/checkout/missing-child",
1604 + ],
1605 + },
1606 +] as const)(
1607 + "does not start provider work in $overlap until the restore finishes",
1608 + ({ restorePath, worktreePath, realPaths, missing }) =>
1609 + Effect.gen(function* () {
1610 + const order: Array<string> = [];
1611 + const releaseRestore = yield* Deferred.make<void>();
1612 + const restoreEntered = yield* Deferred.make<void>();
1613 + const fileSystem = FileSystem.makeNoop({
1614 + exists: () => Effect.succeed(true),
1615 + realPath: (candidate) =>
1616 + (missing as ReadonlyArray<string>).includes(candidate)
1617 + ? Effect.fail(
1618 + PlatformError.systemError({
1619 + _tag: "NotFound",
1620 + module: "FileSystem",
1621 + method: "realPath",
1622 + pathOrDescriptor: candidate,
1623 + }),
1624 + )
1625 + : Effect.succeed((realPaths as Record<string, string>)[candidate] ?? candidate),
1626 + });
1627 + const harness = makeLocalCommandHarness({
1628 + text: "Continue",
1629 + ...(worktreePath === undefined ? {} : { worktreePath }),
1630 + fileSystem,
1631 + failReadsAfterRunning: true,
1632 + });
1633 + const open = harness.open.getMockImplementation()!;
1634 + harness.open.mockImplementation(() =>
1635 + Effect.sync(() => order.push("provider session opened")).pipe(Effect.andThen(open())),
1636 + );
1637 + // The reservation CheckpointRollbackServiceV2.execute holds around its
1638 + // isolation check, provider rollback, and git restore of `restorePath`.
1639 + const restore = yield* withCheckpointRestoreReservation(
1640 + fileSystem,
1641 + yield* Path.Path,
1642 + restorePath,
1643 + Deferred.succeed(restoreEntered, undefined).pipe(
1644 + Effect.andThen(Deferred.await(releaseRestore)),
1645 + Effect.andThen(Effect.sync(() => order.push("checkpoint files restored"))),
1646 + ),
1647 + ).pipe(Effect.forkChild({ startImmediately: true }));
1648 + yield* Deferred.await(restoreEntered);
1649 + const startup = yield* harness.start.pipe(Effect.forkChild({ startImmediately: true }));
1650 + yield* Deferred.succeed(releaseRestore, undefined);
1651 + yield* Fiber.join(restore);
1652 + yield* Fiber.join(startup);
1653 + expect(order).toEqual(["checkpoint files restored", "provider session opened"]);
1654 + }).pipe(Effect.provide(Path.layer)),
1655 +);
1656 +
1657 effectIt.effect("reads current startup state after waiting for the workspace lease", () =>
1658 Effect.gen(function* () {
1659 const worktreePath = "/tmp/provider-start-fresh-state";
apps/server/src/orchestration-v2/ProviderTurnStartService.ts
+17 -3
@@ -30,7 +30,7 @@ import {
30 CLAUDE_PROVIDER,
31 CLAUDE_QUERY_ENDED_BEFORE_TURN_DETAIL,
32 } from "./Adapters/ClaudeAdapterV2.ts";
33 -import { withWorkspaceLease } from "../workspace/workspaceLease.ts";
33 +import { withTurnStartReservation, withWorkspaceLease } from "../workspace/workspaceLease.ts";
34 import * as GitWorkflowService from "../git/GitWorkflowService.ts";
35 import * as ProjectService from "../project/ProjectService.ts";
36 import * as ProviderAuthService from "../provider/ProviderAuthService.ts";
@@ -154,7 +154,9 @@ export const layer: Layer.Layer<
154 const turnAnalytics = yield* ProviderUsageAnalytics.ProviderTurnAnalytics;
155 const contextHandoffService = yield* ContextHandoffService.ContextHandoffServiceV2;
156 const idAllocator = yield* IdAllocator.IdAllocatorV2;
157 + const fileSystem = yield* FileSystem.FileSystem;
158 const path = yield* Path.Path;
159 + const projects = yield* ProjectService.ProjectService;
160 const providerAuth = yield* ProviderAuthService.ProviderAuthService;
161 const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
162 const providerSessions = yield* ProviderSessionManager.ProviderSessionManagerV2;
@@ -1763,9 +1765,21 @@ export const layer: Layer.Layer<
1765 // Storage cleanup removes worktrees under the same lease. Only startup
1766 // holds it; the provider turn runs after it is released.
1767 const { thread } = yield* projectionStore.getThreadRecords(input.threadId, []);
1766 - const providerTurn = yield* thread.worktreePath === null
1768 + const cwd =
1769 + thread.worktreePath ??
1770 + (yield* projects
1771 + .getById(thread.projectId, { includeDeleted: true })
1772 + .pipe(Effect.map((project) => Option.getOrUndefined(project)?.workspaceRoot)));
1773 + const providerTurn = yield* cwd === undefined
1774 ? start(input)
1768 - : withWorkspaceLease(path.resolve(thread.worktreePath), start(input));
1775 + : withTurnStartReservation(
1776 + fileSystem,
1777 + path,
1778 + path.resolve(cwd),
1779 + thread.worktreePath === null
1780 + ? start(input)
1781 + : withWorkspaceLease(path.resolve(thread.worktreePath), start(input)),
1782 + );
1783 if (providerTurn !== undefined) yield* providerTurn;
1784 },
1785 (effect, input) =>
apps/server/src/workspace/workspaceLease.ts
+92
@@ -1,4 +1,7 @@
1 +import * as Deferred from "effect/Deferred";
2 import * as Effect from "effect/Effect";
3 +import type * as FileSystem from "effect/FileSystem";
4 +import type * as Path from "effect/Path";
5 import * as Semaphore from "effect/Semaphore";
6
7 const leases = new Map<string, { semaphore: Semaphore.Semaphore; users: number }>();
@@ -20,3 +23,92 @@ export const withWorkspaceLease = <A, E, R>(
23 ),
24 );
25 });
26 +
27 +// A restore rewrites files under its directory, so it must exclude provider
28 +// startup in any overlapping workspace: symlink aliases, parents, and children.
29 +// Kept apart from withWorkspaceLease because that lease is non-reentrant and
30 +// restores call Git work that leases the project root.
31 +interface Reservation {
32 + readonly cwd: string;
33 + readonly kind: "restore" | "turn-start";
34 +}
35 +
36 +const reservations = new Set<Reservation>();
37 +let reservationReleased = Deferred.makeUnsafe<void>();
38 +
39 +// path.relative folds win32 drive-letter case and treats a POSIX backslash as
40 +// part of a name, unlike a raw prefix check.
41 +const contains = (path: Path.Path, parent: string, child: string) => {
42 + const relative = path.relative(parent, child);
43 + return (
44 + relative === "" ||
45 + (!path.isAbsolute(relative) && relative !== ".." && !relative.startsWith(`..${path.sep}`))
46 + );
47 +};
48 +
49 +const conflicts = (path: Path.Path, a: Reservation, b: Reservation) =>
50 + (a.kind === "restore" || b.kind === "restore") &&
51 + (contains(path, a.cwd, b.cwd) || contains(path, b.cwd, a.cwd));
52 +
53 +const reserve = (path: Path.Path, reservation: Reservation): Effect.Effect<void> =>
54 + Effect.suspend(() => {
55 + for (const held of reservations) {
56 + if (conflicts(path, held, reservation)) {
57 + return Deferred.await(reservationReleased).pipe(Effect.andThen(reserve(path, reservation)));
58 + }
59 + }
60 + reservations.add(reservation);
61 + return Effect.void;
62 + });
63 +
64 +const release = (reservation: Reservation) =>
65 + Effect.sync(() => {
66 + if (!reservations.delete(reservation)) return;
67 + const released = reservationReleased;
68 + reservationReleased = Deferred.makeUnsafe<void>();
69 + Deferred.doneUnsafe(released, Effect.void);
70 + });
71 +
72 +// A worktree removed by storage cleanup is recreated inside the reserved
73 +// section, so resolve its nearest existing ancestor: a symlink alias above it
74 +// must still canonicalize the same on both sides. Walks at most the path depth.
75 +const canonicalize = (
76 + fileSystem: FileSystem.FileSystem,
77 + path: Path.Path,
78 + target: string,
79 +): Effect.Effect<string> =>
80 + fileSystem.realPath(target).pipe(
81 + Effect.catchReason("PlatformError", "NotFound", () => {
82 + const parent = path.dirname(target);
83 + return parent === target
84 + ? Effect.succeed(target)
85 + : canonicalize(fileSystem, path, parent).pipe(
86 + Effect.map((resolved) => path.join(resolved, path.basename(target))),
87 + );
88 + }),
89 + Effect.orElseSucceed(() => target),
90 + );
91 +
92 +const withReservation =
93 + (kind: Reservation["kind"]) =>
94 + <A, E, R>(
95 + fileSystem: FileSystem.FileSystem,
96 + path: Path.Path,
97 + cwd: string,
98 + effect: Effect.Effect<A, E, R>,
99 + ): Effect.Effect<A, E, R> =>
100 + canonicalize(fileSystem, path, cwd).pipe(
101 + Effect.flatMap((canonical) => {
102 + const reservation: Reservation = { cwd: canonical, kind };
103 + return reserve(path, reservation).pipe(
104 + Effect.andThen(effect),
105 + Effect.ensuring(release(reservation)),
106 + );
107 + }),
108 + );
109 +
110 +/** Exclusive over every overlapping workspace while checkpoint files are restored. */
111 +export const withCheckpointRestoreReservation = withReservation("restore");
112 +
113 +/** Waits out overlapping restores; turn startups never wait on each other. */
114 +export const withTurnStartReservation = withReservation("turn-start");