@devries-cameron20 / t3code-1 / commits / 5ce1bd5e14
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): runs no longer stay preparing after a full disk (#28029)

Co-authored-by: Claude Fable 5.2 (primary) <noreply@anthropic.com> Co-authored-by: maria-rcks <maria@kuuro.net>

Alb11747 committed Nov 19, 2026 at 00:57 UTC 5ce1bd5e142f37a4bf985688e90527e57c0d3bf4
2 files changed +195 -29
apps/server/src/orchestration-v2/ThreadLaunchService.test.ts
+117 -1
@@ -20,6 +20,7 @@ import {
20 EnvironmentId,
21 GitCommandError,
22 MessageId,
23 + RunId,
24 ProjectId,
25 ProviderDriverKind,
26 ProviderInstanceId,
@@ -61,6 +62,7 @@ import * as TextGeneration from "../textGeneration/TextGeneration.ts";
62 import { ClaudeProviderCapabilitiesV2 } from "./Adapters/ClaudeAdapterV2.ts";
63 import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
64 import * as CommandReceiptStore from "./CommandReceiptStore.ts";
65 +import * as Orchestrator from "./Orchestrator.ts";
66 import * as EffectOutbox from "./EffectOutbox.ts";
67 import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
68 import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts";
@@ -138,6 +140,10 @@ interface HarnessOptions {
140 readonly getLaunchSettings?: ServerSettings.ServerSettingsService["Service"]["getSettings"];
141 readonly providers?: ReadonlyArray<ServerProvider>;
142 readonly git?: Partial<GitWorkflow.GitWorkflowService["Service"]>;
143 + readonly dispatch?: (
144 + command: Parameters<ThreadManagement.ThreadManagementService["Service"]["dispatch"]>[0],
145 + dispatch: ThreadManagement.ThreadManagementService["Service"]["dispatch"],
146 + ) => ReturnType<ThreadManagement.ThreadManagementService["Service"]["dispatch"]>;
147 readonly isGitRepository?: CheckpointStore.CheckpointStore["Service"]["isGitRepository"];
148 }
149
@@ -247,11 +253,25 @@ function makeHarness(options: HarnessOptions = {}) {
253 folderForThread: () => Effect.succeedNone,
254 }),
255 );
256 + const interceptDispatch = options.dispatch;
257 + const layerLaunchThreads =
258 + interceptDispatch === undefined
259 + ? layerThreadManagement
260 + : Layer.effect(
261 + ThreadManagement.ThreadManagementService,
262 + Effect.gen(function* () {
263 + const threads = yield* ThreadManagement.ThreadManagementService;
264 + return ThreadManagement.ThreadManagementService.of({
265 + ...threads,
266 + dispatch: (command) => interceptDispatch(command, threads.dispatch),
267 + });
268 + }),
269 + ).pipe(Layer.provide(layerThreadManagement));
270 const layerLaunch = ThreadLaunch.layer.pipe(
271 Layer.provide(
272 Layer.mergeAll(
273 layerExternalServices,
254 - layerThreadManagement,
274 + layerLaunchThreads,
275 layerReceipts,
276 layerOrchestrator,
277 IdAllocator.layer,
@@ -1813,6 +1833,102 @@ it.effect.each(["fails", "is interrupted"] as const)(
1833 }),
1834 );
1835
1836 +it.effect("returns from prepareRun without retrying a scheduling failure", () =>
1837 + Effect.gen(function* () {
1838 + const rejected = yield* Deferred.make<void>();
1839 + let rejectedWrites = 0;
1840 + const harness = makeHarness({
1841 + dispatch: (command, dispatch) =>
1842 + command.type !== "prepared-run.fail"
1843 + ? dispatch(command)
1844 + : Effect.suspend(() => {
1845 + rejectedWrites += 1;
1846 + return Deferred.succeed(rejected, undefined).pipe(
1847 + Effect.andThen(Effect.die("SQLITE_FULL: database or disk is full")),
1848 + );
1849 + }),
1850 + });
1851 + yield* Effect.gen(function* () {
1852 + const launches = yield* ThreadLaunch.ThreadLaunchService;
1853 + yield* launches.prepareRun({
1854 + commandId: CommandId.make("command:prepare:missing-thread"),
1855 + threadId: ThreadId.make("thread:prepare:missing-thread"),
1856 + runId: RunId.make("run:prepare:missing-thread"),
1857 + });
1858 + yield* Deferred.await(rejected);
1859 + yield* TestClock.adjust("1 second");
1860 + assert.equal(rejectedWrites, 1);
1861 + }).pipe(Effect.provide(harness.layer));
1862 + }),
1863 +);
1864 +
1865 +it.effect("fails a run whose preparation failure could not be stored until storage recovers", () =>
1866 + Effect.gen(function* () {
1867 + const storageFull = yield* Deferred.make<void>();
1868 + let rejectedWrites = 0;
1869 + let preparations = 0;
1870 + const harness = makeHarness({
1871 + fetchWorktreeBaseCommit: () =>
1872 + Effect.sync(() => preparations++).pipe(
1873 + Effect.andThen(
1874 + Effect.fail(
1875 + new GitCommandError({
1876 + operation: "GitVcsDriver.fetchRemote",
1877 + command: "git",
1878 + cwd: project.workspaceRoot,
1879 + detail: "Git could not reach the remote.",
1880 + exitCode: 128,
1881 + }),
1882 + ),
1883 + ),
1884 + ),
1885 + dispatch: (command, dispatch) =>
1886 + command.type !== "prepared-run.fail"
1887 + ? dispatch(command)
1888 + : Effect.suspend(() => {
1889 + if (rejectedWrites >= 2) return dispatch(command);
1890 + rejectedWrites += 1;
1891 + return Deferred.succeed(storageFull, undefined).pipe(
1892 + Effect.andThen(
1893 + Effect.fail(
1894 + new Orchestrator.OrchestratorDispatchError({
1895 + commandId: command.commandId,
1896 + commandType: command.type,
1897 + cause: new Error("SQLITE_FULL: database or disk is full"),
1898 + }),
1899 + ),
1900 + ),
1901 + );
1902 + }),
1903 + });
1904 + yield* Effect.gen(function* () {
1905 + const launches = yield* ThreadLaunch.ThreadLaunchService;
1906 + const threads = yield* ThreadManagement.ThreadManagementService;
1907 + const input = launchInput({
1908 + command: "command:launch:storage-full",
1909 + thread: "thread:launch:storage-full",
1910 + message: "First message",
1911 + workspace: { type: "worktree", baseRef: "main", startFromOrigin: true },
1912 + });
1913 + const launched = yield* launches.launch(input);
1914 + yield* Deferred.await(storageFull);
1915 + yield* TestClock.adjust("0 seconds");
1916 + yield* launches.launch(input);
1917 + yield* TestClock.adjust("0 seconds");
1918 + assert.equal(preparations, 1);
1919 + yield* TestClock.adjust("1 second");
1920 + yield* threads.streamStoredEventsFrom({ threadId: launched.threadId }).pipe(
1921 + Stream.filter(
1922 + (stored) =>
1923 + stored.event.type === "run.updated" && stored.event.payload.status === "failed",
1924 + ),
1925 + Stream.runHead,
1926 + );
1927 + assert.equal(rejectedWrites, 2);
1928 + }).pipe(Effect.provide(harness.layer));
1929 + }),
1930 +);
1931 +
1932 it.effect.each(["failed", "interrupted"] as const)(
1933 "holds every message that arrives after the setup %s without a worktree",
1934 (ending) => {
apps/server/src/orchestration-v2/ThreadLaunchService.ts
+78 -28
@@ -26,12 +26,14 @@ import * as Cause from "effect/Cause";
26 import * as Clock from "effect/Clock";
27 import * as Context from "effect/Context";
28 import * as DateTime from "effect/DateTime";
29 +import * as Duration from "effect/Duration";
30 import * as Effect from "effect/Effect";
31 import * as Exit from "effect/Exit";
32 import * as Fiber from "effect/Fiber";
33 import * as Layer from "effect/Layer";
34 import * as Option from "effect/Option";
35 import * as Ref from "effect/Ref";
36 +import * as Schedule from "effect/Schedule";
37 import * as Schema from "effect/Schema";
38 import * as Scope from "effect/Scope";
39 import * as Stream from "effect/Stream";
@@ -1110,36 +1112,78 @@ const make = Effect.gen(function* () {
1112 threadId: ThreadId,
1113 runId: RunId | null,
1114 cause: unknown,
1113 - ) =>
1114 - runId === null
1115 - ? Effect.logWarning("Thread workspace preparation failed", {
1116 - commandId: input.commandId,
1117 - threadId,
1118 - cause,
1119 - })
1120 - : threads
1121 - .dispatch({
1122 - type: "prepared-run.fail",
1123 - commandId: CommandId.make(`${input.commandId}:fail`),
1115 + retryPersistence = true,
1116 + ) => {
1117 + if (runId === null) {
1118 + return Effect.logWarning("Thread workspace preparation failed", {
1119 + commandId: input.commandId,
1120 + threadId,
1121 + cause,
1122 + });
1123 + }
1124 + const persistFailure = threads.dispatch({
1125 + type: "prepared-run.fail",
1126 + commandId: CommandId.make(`${input.commandId}:fail`),
1127 + threadId,
1128 + runId,
1129 + failure: makeProviderFailure({
1130 + cause,
1131 + message: failureDetail(cause),
1132 + class: "validation_error",
1133 + retryable: false,
1134 + }),
1135 + });
1136 + const runIsPreparing = threads.getThreadRecords(threadId, ["runs"], { runIds: [runId] }).pipe(
1137 + Effect.map((current) =>
1138 + current.runs.some((run) => run.id === runId && run.status === "preparing"),
1139 + ),
1140 + // The read can fail in the same outage, which says nothing about the run.
1141 + Effect.catchCause(() => Effect.succeed(true)),
1142 + );
1143 + return persistFailure.pipe(
1144 + Effect.catchCause((persistCause) =>
1145 + Effect.gen(function* () {
1146 + yield* Effect.logWarning("Failed to persist thread workspace preparation failure", {
1147 + commandId: input.commandId,
1148 threadId,
1125 - runId,
1126 - failure: makeProviderFailure({
1127 - cause,
1128 - message: failureDetail(cause),
1129 - class: "validation_error",
1130 - retryable: false,
1131 - }),
1132 - })
1133 - .pipe(
1134 - Effect.catchCause((persistCause) =>
1135 - Effect.logWarning("Failed to persist thread workspace preparation failure", {
1136 - commandId: input.commandId,
1137 - threadId,
1138 - cause,
1139 - persistCause,
1140 - }),
1149 + cause,
1150 + persistCause,
1151 + });
1152 + if (
1153 + !retryPersistence ||
1154 + Cause.hasInterruptsOnly(persistCause) ||
1155 + !(yield* runIsPreparing)
1156 + )
1157 + return;
1158 + // A full disk rejects this write as well, and nothing else moves the run out of
1159 + // "preparing", so it would hold the thread's queue until a restart. Retry until
1160 + // storage recovers or the run settles another way.
1161 + yield* persistFailure.pipe(
1162 + // Storage failures can be defects; retry both failure channels.
1163 + Effect.catchCauseIf(
1164 + (retryCause) => !Cause.hasInterruptsOnly(retryCause),
1165 + (retryCause) =>
1166 + runIsPreparing.pipe(
1167 + Effect.flatMap((preparing) =>
1168 + preparing ? Effect.fail(retryCause) : Effect.void,
1169 + ),
1170 + ),
1171 ),
1172 + Effect.retry(
1173 + Schedule.exponential("1 second").pipe(
1174 + Schedule.modifyDelay(({ duration }) =>
1175 + Effect.succeed(Duration.min(duration, Duration.seconds(30))),
1176 + ),
1177 + ),
1178 + ),
1179 + Effect.ignoreCause,
1180 + // Callers reach this from finalizers; shutdown must still be able to stop it.
1181 + Effect.interruptible,
1182 );
1183 + }),
1184 + ),
1185 + );
1186 + };
1187
1188 const reservePreparation = (commandId: CommandId) =>
1189 Ref.modify(scheduledLaunches, (scheduled) => {
@@ -1489,7 +1533,13 @@ const make = Effect.gen(function* () {
1533 );
1534 }).pipe(Effect.exit);
1535 if (Exit.isFailure(scheduled)) {
1492 - yield* failPreparedRun(input, input.threadId, input.runId, Cause.squash(scheduled.cause));
1536 + yield* failPreparedRun(
1537 + input,
1538 + input.threadId,
1539 + input.runId,
1540 + Cause.squash(scheduled.cause),
1541 + false,
1542 + );
1543 }
1544 });
1545