@devries-cameron20 / t3code Mirror · read-only / commits / 0604547a3a
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 15:45 UTC

fix(server): a thread whose worktree was deleted recovers on the next message (#18359)

Co-authored-by: Julius Marminge <julius0216@outlook.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Signed-off-by: Yordis Prieto <yordis.prieto@gmail.com>

Yordis Prieto committed Oct 26, 2026 at 18:28 UTC 0604547a3ae7bca2e5cad1c17b40be5de17dd2cc
4 files changed +338 -56
apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts
+185 -7
@@ -3,6 +3,7 @@ import { expect, it, vi } from "vite-plus/test";
3 import { it as effectIt } from "@effect/vitest";
4 import {
5 CheckpointScopeId,
6 + GitCommandError,
7 MessageId,
8 NodeId,
9 ProviderSessionId,
@@ -33,6 +34,8 @@ import * as Schema from "effect/Schema";
34 import * as ServerConfig from "../config.ts";
35 import { withWorkspaceLease } from "../workspace/workspaceLease.ts";
36 import * as GitWorkflow from "../git/GitWorkflowService.ts";
37 +import * as GitVcsDriver from "../vcs/GitVcsDriver.ts";
38 +import { layerRealGit } from "./ProviderTurnStartService.testkit.ts";
39 import * as ProjectService from "../project/ProjectService.ts";
40 import * as ProviderAuthService from "../provider/ProviderAuthService.ts";
41 import * as ServerSettings from "../serverSettings.ts";
@@ -121,6 +124,7 @@ function startWithMissingWorktree(
124 const startRootRun = vi.fn(() => Effect.void);
125 const pruneWorktrees = vi.fn(() => Effect.void);
126 const createWorktree = vi.fn(() => Effect.succeed({} as never));
127 + const listLocalBranchNames = vi.fn(() => Effect.succeed(["feature/restore"]));
128 const layer = ProviderTurnStart.layer.pipe(
129 Layer.provide(
130 Layer.mergeAll(
@@ -142,7 +146,11 @@ function startWithMissingWorktree(
146 ),
147 } as never),
148 layerWorktreeLocation(),
145 - Layer.mock(GitWorkflow.GitWorkflowService)({ pruneWorktrees, createWorktree }),
149 + Layer.mock(GitWorkflow.GitWorkflowService)({
150 + pruneWorktrees,
151 + createWorktree,
152 + listLocalBranchNames,
153 + }),
154 Layer.mock(ProjectService.ProjectService)({
155 getById: () =>
156 Effect.succeedSome({ workspaceRoot: "/tmp/provider-turn-start-project" } as never),
@@ -257,6 +265,8 @@ function makeLocalCommandHarness(input: {
265 readonly text: string;
266 readonly worktreePath?: string;
267 readonly worktreesDir?: string;
268 + readonly workspaceRoot?: string;
269 + readonly gitWorkflow?: GitWorkflow.GitWorkflowService["Service"];
270 readonly fileSystem?: FileSystem.FileSystem;
271 readonly previousNativeSession?: boolean;
272 readonly previousMessages?: ReadonlyArray<string>;
@@ -664,7 +674,9 @@ function makeLocalCommandHarness(input: {
674 ({ thread }: Parameters<RuntimePolicy.RuntimePolicyV2Shape["resolve"]>[0]) =>
675 Effect.succeed({ cwd: thread.worktreePath ?? "/tmp/native-account-command" } as never),
676 );
667 - const createWorktree = vi.fn(() => Effect.succeed({} as never));
677 + const createWorktree = vi.fn<GitWorkflow.GitWorkflowService["Service"]["createWorktree"]>(() =>
678 + Effect.succeed({} as never),
679 + );
680 const getThreadRecords = vi.fn<ProjectionStore.ProjectionStoreV2Shape["getThreadRecords"]>(() =>
681 Effect.succeed(projection),
682 );
@@ -690,13 +702,18 @@ function makeLocalCommandHarness(input: {
702 input.fileSystem === undefined
703 ? FileSystem.layerNoop({ exists: () => Effect.succeed(true) })
704 : Layer.succeed(FileSystem.FileSystem, input.fileSystem),
693 - Layer.mock(GitWorkflow.GitWorkflowService)({
694 - pruneWorktrees: () => Effect.void,
695 - createWorktree,
696 - }),
705 + input.gitWorkflow === undefined
706 + ? Layer.mock(GitWorkflow.GitWorkflowService)({
707 + pruneWorktrees: () => Effect.void,
708 + listLocalBranchNames: () => Effect.succeed(["feature/restore"]),
709 + createWorktree,
710 + })
711 + : Layer.succeed(GitWorkflow.GitWorkflowService, input.gitWorkflow),
712 Layer.mock(ProjectService.ProjectService)({
713 getById: () =>
699 - Effect.succeedSome({ workspaceRoot: "/tmp/native-account-project" } as never),
714 + Effect.succeedSome({
715 + workspaceRoot: input.workspaceRoot ?? "/tmp/native-account-project",
716 + } as never),
717 }),
718 Layer.mock(ProjectionStore.ProjectionStoreV2)({
719 getThreadRecords,
@@ -770,6 +787,167 @@ function makeLocalCommandHarness(input: {
787 };
788 }
789
790 +effectIt.effect(
791 + "recovers the thread branch tip, falls back to the current project branch, and rejects detached HEAD",
792 + () =>
793 + Effect.gen(function* () {
794 + const fileSystem = yield* FileSystem.FileSystem;
795 + const path = yield* Path.Path;
796 + const gitWorkflow = yield* GitWorkflow.GitWorkflowService;
797 + const driver = yield* GitVcsDriver.GitVcsDriver;
798 + const root = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-recreate-repo-" });
799 + const workspaceRoot = path.join(root, "project");
800 + const worktreesDir = path.join(root, "worktrees");
801 + const worktreePath = path.join(worktreesDir, "project", "restore");
802 + yield* fileSystem.makeDirectory(workspaceRoot);
803 + const git = (cwd: string, args: ReadonlyArray<string>) =>
804 + driver
805 + .execute({ operation: "test.worktreeRecovery", cwd, args })
806 + .pipe(Effect.map((result) => result.stdout.trim()));
807 + yield* git(workspaceRoot, ["init", "-b", "main"]);
808 + yield* git(workspaceRoot, ["config", "user.name", "Test"]);
809 + yield* git(workspaceRoot, ["config", "user.email", "test@example.com"]);
810 + yield* git(workspaceRoot, ["commit", "--allow-empty", "-m", "base"]);
811 + yield* git(workspaceRoot, ["worktree", "add", "-b", "feature/restore", worktreePath]);
812 + yield* fileSystem.writeFileString(path.join(worktreePath, "thread.txt"), "thread work");
813 + yield* git(worktreePath, ["add", "."]);
814 + yield* git(worktreePath, ["commit", "-m", "thread tip"]);
815 + const threadTip = yield* git(worktreePath, ["rev-parse", "HEAD"]);
816 + yield* git(workspaceRoot, ["switch", "-c", "project-current"]);
817 + yield* fileSystem.writeFileString(
818 + path.join(workspaceRoot, "project.txt"),
819 + "new project code",
820 + );
821 + yield* git(workspaceRoot, ["add", "."]);
822 + yield* git(workspaceRoot, ["commit", "-m", "project tip"]);
823 + const projectTip = yield* git(workspaceRoot, ["rev-parse", "HEAD"]);
824 + expect(projectTip).not.toBe(threadTip);
825 + // Tags must neither hide the thread branch nor select a different recovery tip.
826 + yield* git(workspaceRoot, ["tag", "feature/restore", projectTip]);
827 + yield* git(workspaceRoot, ["tag", "project-current", threadTip]);
828 +
829 + // Reuse the workflow across turns so recovery cannot select a cached project branch.
830 + for (const scenario of ["retained", "deleted", "detached"] as const) {
831 + yield* fileSystem.remove(worktreePath, { recursive: true });
832 + if (scenario !== "retained") {
833 + yield* git(workspaceRoot, ["worktree", "prune"]);
834 + yield* git(workspaceRoot, ["branch", "-D", "feature/restore"]);
835 + }
836 + if (scenario === "detached") {
837 + yield* git(workspaceRoot, ["switch", "--detach"]);
838 + expect(yield* git(workspaceRoot, ["branch", "--show-current"])).toBe("");
839 + }
840 + const harness = makeLocalCommandHarness({
841 + text: "Continue",
842 + workspaceRoot,
843 + worktreePath,
844 + worktreesDir,
845 + fileSystem,
846 + gitWorkflow,
847 + failReadsAfterRunning: true,
848 + });
849 + const open = harness.open.getMockImplementation()!;
850 + harness.open.mockImplementation(() =>
851 + Effect.gen(function* () {
852 + expect(yield* git(worktreePath, ["branch", "--show-current"])).toBe("feature/restore");
853 + expect(yield* git(worktreePath, ["rev-parse", "HEAD"])).toBe(
854 + scenario === "retained" ? threadTip : projectTip,
855 + );
856 + expect(
857 + yield* fileSystem.readFileString(
858 + path.join(worktreePath, scenario === "retained" ? "thread.txt" : "project.txt"),
859 + ),
860 + ).toBe(scenario === "retained" ? "thread work" : "new project code");
861 + return yield* open();
862 + }).pipe(Effect.orDie),
863 + );
864 +
865 + yield* harness.start;
866 +
867 + if (scenario === "detached") {
868 + expect(harness.open).not.toHaveBeenCalled();
869 + expect(harness.startRootRun).not.toHaveBeenCalled();
870 + expect(harness.projection().runs.at(-1)?.status).toBe("failed");
871 + expect(harness.projection().turnItems).toMatchObject([
872 + { type: "error", failure: { message: expect.stringContaining("no current branch") } },
873 + ]);
874 + expect(yield* fileSystem.exists(worktreePath)).toBe(false);
875 + expect(yield* git(workspaceRoot, ["branch", "--list", "feature/restore"])).toBe("");
876 + } else {
877 + expect(harness.open).toHaveBeenCalledOnce();
878 + expect(harness.startRootRun).toHaveBeenCalledWith(
879 + expect.objectContaining({
880 + runtimePolicy: expect.objectContaining({ cwd: worktreePath }),
881 + }),
882 + );
883 + expect(harness.projection().runs.at(-1)?.status).toBe("running");
884 + }
885 + }
886 + }).pipe(Effect.scoped, Effect.provide(layerRealGit)),
887 +);
888 +
889 +effectIt.effect(
890 + "preserves recreation interrupts and retries, then fails the final attempt before provider start",
891 + () =>
892 + Effect.gen(function* () {
893 + const fileSystem = yield* FileSystem.FileSystem;
894 + const path = yield* Path.Path;
895 + const worktreesDir = yield* fileSystem.makeTempDirectoryScoped({
896 + prefix: "t3-recreate-failure-",
897 + });
898 + const worktreePath = path.join(worktreesDir, "project", "restore");
899 + const harness = makeLocalCommandHarness({
900 + text: "Continue",
901 + worktreesDir,
902 + worktreePath,
903 + fileSystem,
904 + failReadsAfterRunning: true,
905 + });
906 + harness.createWorktree.mockImplementationOnce(() => Effect.interrupt);
907 + harness.createWorktree.mockImplementation(() =>
908 + Effect.fail(
909 + new GitCommandError({
910 + operation: "worktree-add",
911 + command: "git worktree add",
912 + cwd: "/tmp/native-account-project",
913 + detail: "fatal: private git diagnostic",
914 + }),
915 + ),
916 + );
917 +
918 + const interrupted = yield* Effect.exit(harness.start);
919 + expect(interrupted._tag).toBe("Failure");
920 + if (interrupted._tag === "Failure")
921 + expect(Cause.hasInterruptsOnly(interrupted.cause)).toBe(true);
922 + expect(harness.events).toEqual([]);
923 + expect(harness.projection().runs.at(-1)?.status).toBe("starting");
924 +
925 + const retry = yield* Effect.result(harness.startWithRetry);
926 + expect(retry).toMatchObject({ _tag: "Failure", failure: { _tag: "ProviderTurnStartError" } });
927 + expect(harness.events).toEqual([]);
928 + expect(harness.projection().runs.at(-1)?.status).toBe("starting");
929 +
930 + yield* harness.start;
931 + expect(harness.createWorktree).toHaveBeenCalledTimes(3);
932 + expect(harness.writeIfRunCurrent).toHaveBeenCalledOnce();
933 + expect(harness.open).not.toHaveBeenCalled();
934 + expect(harness.startRootRun).not.toHaveBeenCalled();
935 + const projection = harness.projection();
936 + expect(projection.runs.at(-1)).toMatchObject({ status: "failed", startedAt: null });
937 + expect(projection.attempts[0]).toMatchObject({ status: "failed", startedAt: null });
938 + expect(projection.nodes[0]).toMatchObject({ status: "failed", startedAt: null });
939 + expect(projection.turnItems).toMatchObject([
940 + {
941 + type: "error",
942 + failure: {
943 + message: `Could not recreate the worktree for branch 'feature/restore' at ${worktreePath}.`,
944 + },
945 + },
946 + ]);
947 + expect(JSON.stringify(harness.events)).not.toContain("private git diagnostic");
948 + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
949 +);
950 +
951 effectIt.effect("recovers an abandoned failed turn before continuing its native conversation", () =>
952 Effect.gen(function* () {
953 const harness = makeLocalCommandHarness({
apps/server/src/orchestration-v2/ProviderTurnStartService.testkit.ts
+47
@@ -1,3 +1,18 @@
1 +import { SourceControlProviderError, SourceControlProviderKind } from "@t3tools/contracts";
2 +import * as NodeServices from "@effect/platform-node/NodeServices";
3 +import * as ServerConfig from "../config.ts";
4 +import * as GitManager from "../git/GitManager.ts";
5 +import * as GitVcsDriver from "../vcs/GitVcsDriver.ts";
6 +import * as VcsDriverRegistry from "../vcs/VcsDriverRegistry.ts";
7 +import * as VcsProjectConfig from "../vcs/VcsProjectConfig.ts";
8 +import * as VcsProcess from "../vcs/VcsProcess.ts";
9 +import * as ServerSettings from "../serverSettings.ts";
10 +import * as ProjectSetupScriptRunner from "../project/ProjectSetupScriptRunner.ts";
11 +import * as ProviderRegistry from "../provider/ProviderRegistry.ts";
12 +import * as SourceControlProviderRegistry from "../sourceControl/SourceControlProviderRegistry.ts";
13 +import * as TextGeneration from "../textGeneration/TextGeneration.ts";
14 +import * as ProjectionStore from "./ProjectionStore.ts";
15 +import * as ProjectStore from "./ProjectStore.ts";
16 import * as Effect from "effect/Effect";
17 import * as Layer from "effect/Layer";
18
@@ -13,3 +28,35 @@ export const layer = Layer.merge(
28 getById: () => Effect.succeedNone,
29 }),
30 );
31 +
32 +// Keep repository detection, branch selection and worktree creation on the real Git path.
33 +export const layerRealGit = GitWorkflow.layer.pipe(
34 + Layer.provide(GitManager.layer),
35 + Layer.provide(
36 + Layer.mergeAll(
37 + Layer.mock(SourceControlProviderRegistry.SourceControlProviderRegistry)({
38 + resolveLink: () => undefined,
39 + resolve: ({ cwd }) =>
40 + Effect.fail(
41 + new SourceControlProviderError({
42 + provider: SourceControlProviderKind.make("unknown"),
43 + operation: "resolve",
44 + cwd,
45 + detail: "No source control provider in tests",
46 + }),
47 + ),
48 + }),
49 + Layer.mock(TextGeneration.TextGeneration)({}),
50 + Layer.mock(ProviderRegistry.ProviderRegistry)({}),
51 + Layer.mock(ProjectSetupScriptRunner.ProjectSetupScriptRunner)({}),
52 + Layer.mock(ProjectionStore.ProjectionStoreV2)({}),
53 + Layer.mock(ProjectStore.ProjectStoreV2)({}),
54 + ServerSettings.layerTest(),
55 + ),
56 + ),
57 + Layer.provide(VcsDriverRegistry.layer.pipe(Layer.provide(VcsProjectConfig.layer))),
58 + Layer.provideMerge(GitVcsDriver.layer),
59 + Layer.provideMerge(VcsProcess.layer),
60 + Layer.provideMerge(ServerConfig.layerTest(process.cwd(), { prefix: "t3-recreate-" })),
61 + Layer.provideMerge(NodeServices.layer),
62 +);
apps/server/src/orchestration-v2/ProviderTurnStartService.ts
+105 -48
@@ -15,7 +15,6 @@ import {
15 ThreadId,
16 } from "@t3tools/contracts";
17 import * as Context from "effect/Context";
18 -import * as Cause from "effect/Cause";
18 import * as DateTime from "effect/DateTime";
19 import * as Effect from "effect/Effect";
20 import * as FileSystem from "effect/FileSystem";
@@ -68,6 +67,28 @@ export class ProviderTurnStartError extends Schema.TaggedError<ProviderTurnStart
67 },
68 ) {}
69
70 +class WorktreeRecreateError extends Schema.TaggedError<WorktreeRecreateError>()(
71 + "WorktreeRecreateError",
72 + {
73 + worktreePath: Schema.String,
74 + branch: Schema.String,
75 + cause: Schema.Defect(),
76 + },
77 +) {
78 + override get message(): string {
79 + return `Could not recreate the worktree for branch '${this.branch}' at ${this.worktreePath}.`;
80 + }
81 +}
82 +
83 +class WorktreeRecreateNoBaseError extends Schema.TaggedError<WorktreeRecreateNoBaseError>()(
84 + "WorktreeRecreateNoBaseError",
85 + { worktreePath: Schema.String, branch: Schema.String },
86 +) {
87 + override get message(): string {
88 + return `Could not recreate branch '${this.branch}' at ${this.worktreePath}: the project workspace has no current branch to start it from (detached HEAD?).`;
89 + }
90 +}
91 +
92 const isProviderTurnStartError = Schema.is(ProviderTurnStartError);
93
94 /**
@@ -131,6 +152,35 @@ export const layer: Layer.Layer<
152 const runtimePolicy = yield* RuntimePolicy.RuntimePolicyV2;
153 const isManagedWorktreePath = yield* makeIsManagedWorktreePath;
154
155 + const recreateMissingWorktree = Effect.fn("ProviderTurnStartService.recreateMissingWorktree")(
156 + function* (input: {
157 + readonly cwd: string;
158 + readonly worktreePath: string;
159 + readonly branch: string;
160 + }) {
161 + const { cwd, worktreePath, branch } = input;
162 + yield* gitWorkflow.pruneWorktrees({ cwd });
163 + const localBranchNames = yield* gitWorkflow.listLocalBranchNames(cwd);
164 + if (localBranchNames.includes(branch)) {
165 + yield* gitWorkflow.createWorktree({ cwd, refName: branch, path: worktreePath });
166 + return;
167 + }
168 + // Recovery can follow an external checkout, before the status cache expires.
169 + yield* gitWorkflow.invalidateLocalStatus(cwd);
170 + const { refName: baseRef } = yield* gitWorkflow.localStatus({ cwd });
171 + if (baseRef === null) {
172 + return yield* new WorktreeRecreateNoBaseError({ worktreePath, branch });
173 + }
174 + yield* gitWorkflow.createWorktree({
175 + cwd,
176 + refName: `refs/heads/${baseRef}`,
177 + newRefName: branch,
178 + baseRefName: `refs/heads/${baseRef}`,
179 + path: worktreePath,
180 + });
181 + },
182 + );
183 +
184 // Build draining callbacks outside startup scope so they cannot retain full thread history.
185 const makeRunControls = (input: {
186 readonly threadId: ThreadId;
@@ -399,6 +449,39 @@ export const layer: Layer.Layer<
449 });
450 },
451 );
452 + const settleStartFailure = Effect.fn("ProviderTurnStartService.settleStartFailure")(
453 + function* (failed: {
454 + readonly signal: string;
455 + readonly title: string;
456 + readonly error: Error;
457 + readonly message?: string;
458 + }) {
459 + const nestedCause = "cause" in failed.error ? failed.error.cause : undefined;
460 + yield* settleRunBeforeStart({
461 + signal: failed.signal,
462 + status: "failed",
463 + now: yield* DateTime.now,
464 + providerInstanceId: run.providerInstanceId,
465 + itemProviderThreadId: providerThread.id,
466 + item: {
467 + type: "error",
468 + title: failed.title,
469 + failure: makeProviderFailure({
470 + cause: failed.error,
471 + message:
472 + failed.message ??
473 + knownProviderFailureMessage(failed.error) ??
474 + (nestedCause instanceof Error
475 + ? nestedCause.message
476 + : typeof nestedCause === "string"
477 + ? nestedCause
478 + : failed.error.message),
479 + class: "provider_error",
480 + }),
481 + },
482 + });
483 + },
484 + );
485 if (message.attachments.length === 0 && message.text.trimStart().startsWith("/")) {
486 const isEmptyCompaction =
487 message.text.trim().toLowerCase() === "/compact" && !projection.hasConversation;
@@ -500,24 +583,29 @@ export const layer: Layer.Layer<
583 worktreePath,
584 branch,
585 });
503 - yield* gitWorkflow.pruneWorktrees({ cwd: project.workspaceRoot }).pipe(
504 - Effect.andThen(
505 - gitWorkflow.createWorktree({
506 - cwd: project.workspaceRoot,
507 - refName: branch,
508 - path: worktreePath,
509 - }),
510 - ),
511 - Effect.catchCauseIf(
512 - (cause) => !Cause.hasInterruptsOnly(cause),
513 - (cause) =>
514 - Effect.logWarning("provider turn start failed to recreate worktree", {
515 - threadId: projection.thread.id,
516 - worktreePath,
517 - cause: Cause.pretty(cause),
518 - }),
586 + const recreateResult = yield* Effect.result(
587 + recreateMissingWorktree({
588 + cwd: project.workspaceRoot,
589 + worktreePath,
590 + branch,
591 + }).pipe(
592 + Effect.mapError((cause) =>
593 + cause._tag === "WorktreeRecreateNoBaseError"
594 + ? cause
595 + : new WorktreeRecreateError({ worktreePath, branch, cause }),
596 + ),
597 ),
598 );
599 + if (recreateResult._tag === "Failure") {
600 + if (input.willRetry === true) return yield* recreateResult.failure;
601 + yield* settleStartFailure({
602 + signal: "worktree-recreate-failure",
603 + title: "Couldn't recreate thread workspace",
604 + error: recreateResult.failure,
605 + message: recreateResult.failure.message,
606 + });
607 + return;
608 + }
609 }
610 }
611 }
@@ -553,37 +641,6 @@ export const layer: Layer.Layer<
641 const existingSessionProjection = projection.providerSessions.find(
642 (candidate) => candidate.id === providerSessionId,
643 );
556 - const settleStartFailure = Effect.fn("ProviderTurnStartService.settleStartFailure")(
557 - function* (failed: {
558 - readonly signal: string;
559 - readonly title: string;
560 - readonly error: Error;
561 - }) {
562 - const nestedCause = "cause" in failed.error ? failed.error.cause : undefined;
563 - yield* settleRunBeforeStart({
564 - signal: failed.signal,
565 - status: "failed",
566 - now: yield* DateTime.now,
567 - providerInstanceId: run.providerInstanceId,
568 - itemProviderThreadId: providerThread.id,
569 - item: {
570 - type: "error",
571 - title: failed.title,
572 - failure: makeProviderFailure({
573 - cause: failed.error,
574 - message:
575 - knownProviderFailureMessage(failed.error) ??
576 - (nestedCause instanceof Error
577 - ? nestedCause.message
578 - : typeof nestedCause === "string"
579 - ? nestedCause
580 - : failed.error.message),
581 - class: "provider_error",
582 - }),
583 - },
584 - });
585 - },
586 - );
644 const failedRuns = new Map(
645 projection.runs
646 .filter(
apps/server/src/vcs/GitVcsDriverCore.ts
+1 -1
@@ -4385,7 +4385,7 @@ export const makeGitVcsDriverCore = Effect.fn("makeGitVcsDriverCore")(function*
4385 "branch",
4386 "--list",
4387 "--no-column",
4388 - "--format=%(refname:short)",
4388 + "--format=%(refname:lstrip=2)",
4389 ]).pipe(
4390 Effect.map((stdout) => {
4391 const branchNames: Array<string> = [];