The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 07:15 UTC
fix(server): runs no longer stay preparing after a full disk (#28029)
Co-authored-by: Claude Fable 5.2 (primary) <noreply@anthropic.com> Co-authored-by: maria-rcks <maria@kuuro.net>
Alb11747 committed
Nov 19, 2026 at 00:57 UTC
5ce1bd5e142f37a4bf985688e90527e57c0d3bf4
2 files changed
+195
-29
apps/server/src/orchestration-v2/ThreadLaunchService.test.ts
+117
-1
@@ -20,6 +20,7 @@ import {
20
EnvironmentId,
21
GitCommandError,
22
MessageId,
23
+ RunId,
24
ProjectId,
25
ProviderDriverKind,
26
ProviderInstanceId,
@@ -61,6 +62,7 @@ import * as TextGeneration from "../textGeneration/TextGeneration.ts";
62
import { ClaudeProviderCapabilitiesV2 } from "./Adapters/ClaudeAdapterV2.ts";
63
import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
64
import * as CommandReceiptStore from "./CommandReceiptStore.ts";
65
+import * as Orchestrator from "./Orchestrator.ts";
66
import * as EffectOutbox from "./EffectOutbox.ts";
67
import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
68
import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts";
@@ -138,6 +140,10 @@ interface HarnessOptions {
140
readonly getLaunchSettings?: ServerSettings.ServerSettingsService["Service"]["getSettings"];
141
readonly providers?: ReadonlyArray<ServerProvider>;
142
readonly git?: Partial<GitWorkflow.GitWorkflowService["Service"]>;
143
+ readonly dispatch?: (
144
+ command: Parameters<ThreadManagement.ThreadManagementService["Service"]["dispatch"]>[0],
145
+ dispatch: ThreadManagement.ThreadManagementService["Service"]["dispatch"],
146
+ ) => ReturnType<ThreadManagement.ThreadManagementService["Service"]["dispatch"]>;
147
readonly isGitRepository?: CheckpointStore.CheckpointStore["Service"]["isGitRepository"];
148
}
149
@@ -247,11 +253,25 @@ function makeHarness(options: HarnessOptions = {}) {
253
folderForThread: () => Effect.succeedNone,
254
}),
255
);
256
+ const interceptDispatch = options.dispatch;
257
+ const layerLaunchThreads =
258
+ interceptDispatch === undefined
259
+ ? layerThreadManagement
260
+ : Layer.effect(
261
+ ThreadManagement.ThreadManagementService,
262
+ Effect.gen(function* () {
263
+ const threads = yield* ThreadManagement.ThreadManagementService;
264
+ return ThreadManagement.ThreadManagementService.of({
265
+ ...threads,
266
+ dispatch: (command) => interceptDispatch(command, threads.dispatch),
267
+ });
268
+ }),
269
+ ).pipe(Layer.provide(layerThreadManagement));
270
const layerLaunch = ThreadLaunch.layer.pipe(
271
Layer.provide(
272
Layer.mergeAll(
273
layerExternalServices,
254
- layerThreadManagement,
274
+ layerLaunchThreads,
275
layerReceipts,
276
layerOrchestrator,
277
IdAllocator.layer,
@@ -1813,6 +1833,102 @@ it.effect.each(["fails", "is interrupted"] as const)(
1833
}),
1834
);
1835
1836
+it.effect("returns from prepareRun without retrying a scheduling failure", () =>
1837
+ Effect.gen(function* () {
1838
+ const rejected = yield* Deferred.make<void>();
1839
+ let rejectedWrites = 0;
1840
+ const harness = makeHarness({
1841
+ dispatch: (command, dispatch) =>
1842
+ command.type !== "prepared-run.fail"
1843
+ ? dispatch(command)
1844
+ : Effect.suspend(() => {
1845
+ rejectedWrites += 1;
1846
+ return Deferred.succeed(rejected, undefined).pipe(
1847
+ Effect.andThen(Effect.die("SQLITE_FULL: database or disk is full")),
1848
+ );
1849
+ }),
1850
+ });
1851
+ yield* Effect.gen(function* () {
1852
+ const launches = yield* ThreadLaunch.ThreadLaunchService;
1853
+ yield* launches.prepareRun({
1854
+ commandId: CommandId.make("command:prepare:missing-thread"),
1855
+ threadId: ThreadId.make("thread:prepare:missing-thread"),
1856
+ runId: RunId.make("run:prepare:missing-thread"),
1857
+ });
1858
+ yield* Deferred.await(rejected);
1859
+ yield* TestClock.adjust("1 second");
1860
+ assert.equal(rejectedWrites, 1);
1861
+ }).pipe(Effect.provide(harness.layer));
1862
+ }),
1863
+);
1864
+
1865
+it.effect("fails a run whose preparation failure could not be stored until storage recovers", () =>
1866
+ Effect.gen(function* () {
1867
+ const storageFull = yield* Deferred.make<void>();
1868
+ let rejectedWrites = 0;
1869
+ let preparations = 0;
1870
+ const harness = makeHarness({
1871
+ fetchWorktreeBaseCommit: () =>
1872
+ Effect.sync(() => preparations++).pipe(
1873
+ Effect.andThen(
1874
+ Effect.fail(
1875
+ new GitCommandError({
1876
+ operation: "GitVcsDriver.fetchRemote",
1877
+ command: "git",
1878
+ cwd: project.workspaceRoot,
1879
+ detail: "Git could not reach the remote.",
1880
+ exitCode: 128,
1881
+ }),
1882
+ ),
1883
+ ),
1884
+ ),
1885
+ dispatch: (command, dispatch) =>
1886
+ command.type !== "prepared-run.fail"
1887
+ ? dispatch(command)
1888
+ : Effect.suspend(() => {
1889
+ if (rejectedWrites >= 2) return dispatch(command);
1890
+ rejectedWrites += 1;
1891
+ return Deferred.succeed(storageFull, undefined).pipe(
1892
+ Effect.andThen(
1893
+ Effect.fail(
1894
+ new Orchestrator.OrchestratorDispatchError({
1895
+ commandId: command.commandId,
1896
+ commandType: command.type,
1897
+ cause: new Error("SQLITE_FULL: database or disk is full"),
1898
+ }),
1899
+ ),
1900
+ ),
1901
+ );
1902
+ }),
1903
+ });
1904
+ yield* Effect.gen(function* () {
1905
+ const launches = yield* ThreadLaunch.ThreadLaunchService;
1906
+ const threads = yield* ThreadManagement.ThreadManagementService;
1907
+ const input = launchInput({
1908
+ command: "command:launch:storage-full",
1909
+ thread: "thread:launch:storage-full",
1910
+ message: "First message",
1911
+ workspace: { type: "worktree", baseRef: "main", startFromOrigin: true },
1912
+ });
1913
+ const launched = yield* launches.launch(input);
1914
+ yield* Deferred.await(storageFull);
1915
+ yield* TestClock.adjust("0 seconds");
1916
+ yield* launches.launch(input);
1917
+ yield* TestClock.adjust("0 seconds");
1918
+ assert.equal(preparations, 1);
1919
+ yield* TestClock.adjust("1 second");
1920
+ yield* threads.streamStoredEventsFrom({ threadId: launched.threadId }).pipe(
1921
+ Stream.filter(
1922
+ (stored) =>
1923
+ stored.event.type === "run.updated" && stored.event.payload.status === "failed",
1924
+ ),
1925
+ Stream.runHead,
1926
+ );
1927
+ assert.equal(rejectedWrites, 2);
1928
+ }).pipe(Effect.provide(harness.layer));
1929
+ }),
1930
+);
1931
+
1932
it.effect.each(["failed", "interrupted"] as const)(
1933
"holds every message that arrives after the setup %s without a worktree",
1934
(ending) => {
apps/server/src/orchestration-v2/ThreadLaunchService.ts
+78
-28
@@ -26,12 +26,14 @@ import * as Cause from "effect/Cause";
26
import * as Clock from "effect/Clock";
27
import * as Context from "effect/Context";
28
import * as DateTime from "effect/DateTime";
29
+import * as Duration from "effect/Duration";
30
import * as Effect from "effect/Effect";
31
import * as Exit from "effect/Exit";
32
import * as Fiber from "effect/Fiber";
33
import * as Layer from "effect/Layer";
34
import * as Option from "effect/Option";
35
import * as Ref from "effect/Ref";
36
+import * as Schedule from "effect/Schedule";
37
import * as Schema from "effect/Schema";
38
import * as Scope from "effect/Scope";
39
import * as Stream from "effect/Stream";
@@ -1110,36 +1112,78 @@ const make = Effect.gen(function* () {
1112
threadId: ThreadId,
1113
runId: RunId | null,
1114
cause: unknown,
1113
- ) =>
1114
- runId === null
1115
- ? Effect.logWarning("Thread workspace preparation failed", {
1116
- commandId: input.commandId,
1117
- threadId,
1118
- cause,
1119
- })
1120
- : threads
1121
- .dispatch({
1122
- type: "prepared-run.fail",
1123
- commandId: CommandId.make(`${input.commandId}:fail`),
1115
+ retryPersistence = true,
1116
+ ) => {
1117
+ if (runId === null) {
1118
+ return Effect.logWarning("Thread workspace preparation failed", {
1119
+ commandId: input.commandId,
1120
+ threadId,
1121
+ cause,
1122
+ });
1123
+ }
1124
+ const persistFailure = threads.dispatch({
1125
+ type: "prepared-run.fail",
1126
+ commandId: CommandId.make(`${input.commandId}:fail`),
1127
+ threadId,
1128
+ runId,
1129
+ failure: makeProviderFailure({
1130
+ cause,
1131
+ message: failureDetail(cause),
1132
+ class: "validation_error",
1133
+ retryable: false,
1134
+ }),
1135
+ });
1136
+ const runIsPreparing = threads.getThreadRecords(threadId, ["runs"], { runIds: [runId] }).pipe(
1137
+ Effect.map((current) =>
1138
+ current.runs.some((run) => run.id === runId && run.status === "preparing"),
1139
+ ),
1140
+ // The read can fail in the same outage, which says nothing about the run.
1141
+ Effect.catchCause(() => Effect.succeed(true)),
1142
+ );
1143
+ return persistFailure.pipe(
1144
+ Effect.catchCause((persistCause) =>
1145
+ Effect.gen(function* () {
1146
+ yield* Effect.logWarning("Failed to persist thread workspace preparation failure", {
1147
+ commandId: input.commandId,
1148
threadId,
1125
- runId,
1126
- failure: makeProviderFailure({
1127
- cause,
1128
- message: failureDetail(cause),
1129
- class: "validation_error",
1130
- retryable: false,
1131
- }),
1132
- })
1133
- .pipe(
1134
- Effect.catchCause((persistCause) =>
1135
- Effect.logWarning("Failed to persist thread workspace preparation failure", {
1136
- commandId: input.commandId,
1137
- threadId,
1138
- cause,
1139
- persistCause,
1140
- }),
1149
+ cause,
1150
+ persistCause,
1151
+ });
1152
+ if (
1153
+ !retryPersistence ||
1154
+ Cause.hasInterruptsOnly(persistCause) ||
1155
+ !(yield* runIsPreparing)
1156
+ )
1157
+ return;
1158
+ // A full disk rejects this write as well, and nothing else moves the run out of
1159
+ // "preparing", so it would hold the thread's queue until a restart. Retry until
1160
+ // storage recovers or the run settles another way.
1161
+ yield* persistFailure.pipe(
1162
+ // Storage failures can be defects; retry both failure channels.
1163
+ Effect.catchCauseIf(
1164
+ (retryCause) => !Cause.hasInterruptsOnly(retryCause),
1165
+ (retryCause) =>
1166
+ runIsPreparing.pipe(
1167
+ Effect.flatMap((preparing) =>
1168
+ preparing ? Effect.fail(retryCause) : Effect.void,
1169
+ ),
1170
+ ),
1171
),
1172
+ Effect.retry(
1173
+ Schedule.exponential("1 second").pipe(
1174
+ Schedule.modifyDelay(({ duration }) =>
1175
+ Effect.succeed(Duration.min(duration, Duration.seconds(30))),
1176
+ ),
1177
+ ),
1178
+ ),
1179
+ Effect.ignoreCause,
1180
+ // Callers reach this from finalizers; shutdown must still be able to stop it.
1181
+ Effect.interruptible,
1182
);
1183
+ }),
1184
+ ),
1185
+ );
1186
+ };
1187
1188
const reservePreparation = (commandId: CommandId) =>
1189
Ref.modify(scheduledLaunches, (scheduled) => {
@@ -1489,7 +1533,13 @@ const make = Effect.gen(function* () {
1533
);
1534
}).pipe(Effect.exit);
1535
if (Exit.isFailure(scheduled)) {
1492
- yield* failPreparedRun(input, input.threadId, input.runId, Cause.squash(scheduled.cause));
1536
+ yield* failPreparedRun(
1537
+ input,
1538
+ input.threadId,
1539
+ input.runId,
1540
+ Cause.squash(scheduled.cause),
1541
+ false,
1542
+ );
1543
}
1544
});
1545