The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 18:15 UTC
fix(server): retry provider event writes after brief SQLite locks (#24571)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Julius Marminge committed
Oct 24, 2026 at 17:17 UTC
97548db62dd68a1859dd15fabb957e3ae746f65f
2 files changed
+171
-24
apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts
+158
-20
@@ -17,20 +17,23 @@ import {
17
type OrchestrationV2TurnItem,
18
ProviderDriverKind,
19
ProviderInstanceId,
20
- type ProviderThreadId,
20
+ ProviderThreadId,
21
ProviderTurnId,
22
RunAttemptId,
23
RunId,
24
RuntimeRequestId,
25
TurnItemId,
26
} from "@t3tools/contracts";
27
+import * as Cause from "effect/Cause";
28
import * as DateTime from "effect/DateTime";
29
import * as Deferred from "effect/Deferred";
30
import * as Fiber from "effect/Fiber";
31
import * as Effect from "effect/Effect";
32
+import * as Exit from "effect/Exit";
33
import * as FileSystem from "effect/FileSystem";
34
import * as Layer from "effect/Layer";
35
import * as SqlClient from "effect/sql/SqlClient";
36
+import * as SqlError from "effect/sql/SqlError";
37
import * as Stream from "effect/Stream";
38
39
import { toolOutputImages } from "@t3tools/shared/toolOutput";
@@ -65,25 +68,63 @@ const layerGit = GitVcsDriver.layer.pipe(
68
Layer.provideMerge(NodeServices.layer),
69
);
70
68
-const layerTest = Layer.mergeAll(
69
- layerTestDatabase,
70
- layerTestStores,
71
- layerTestEventSink,
72
- IdAllocator.layer,
73
- ThreadCommandExecutor.layer,
74
- layerGit,
75
- ProviderEventIngestor.layer.pipe(
76
- Layer.provide(
77
- Layer.mergeAll(
78
- layerTestStores,
79
- layerTestEventSink,
80
- IdAllocator.layer,
81
- ThreadCommandExecutor.layer,
82
- layerGit,
71
+const makeLayerTest = <E>(ingestorEventSink: Layer.Layer<EventSink.EventSinkV2, E>) =>
72
+ Layer.mergeAll(
73
+ layerTestDatabase,
74
+ layerTestStores,
75
+ layerTestEventSink,
76
+ IdAllocator.layer,
77
+ ThreadCommandExecutor.layer,
78
+ layerGit,
79
+ ProviderEventIngestor.layer.pipe(
80
+ Layer.provide(
81
+ Layer.mergeAll(
82
+ layerTestStores,
83
+ ingestorEventSink,
84
+ IdAllocator.layer,
85
+ ThreadCommandExecutor.layer,
86
+ layerGit,
87
+ ),
88
),
89
),
85
- ),
86
-);
90
+ );
91
+const layerTest = makeLayerTest(layerTestEventSink);
92
+
93
+// EventSink reports a lock that outlasted busy_timeout at BEGIN IMMEDIATE this way.
94
+const lockTimeout = new EventSink.EventSinkWriteError({
95
+ eventCount: 1,
96
+ cause: new SqlError.SqlError({ reason: new SqlError.LockTimeoutError({ cause: null }) }),
97
+});
98
+const constraintFailure = new EventSink.EventSinkWriteError({
99
+ eventCount: 1,
100
+ cause: new SqlError.SqlError({ reason: new SqlError.ConstraintError({ cause: null }) }),
101
+});
102
+
103
+/** The ingestor's next sink writes of `events` fail with `errors`, in order; the rest go through. */
104
+const layerFailingWrites = (
105
+ errors: ReadonlyArray<EventSink.EventSinkWriteError>,
106
+ matches: (events: ReadonlyArray<OrchestrationV2DomainEvent>) => boolean = () => true,
107
+) =>
108
+ makeLayerTest(
109
+ Layer.effect(
110
+ EventSink.EventSinkV2,
111
+ Effect.gen(function* () {
112
+ const eventSink = yield* EventSink.EventSinkV2;
113
+ const pending = [...errors];
114
+ const failNext = <A, R>(
115
+ events: ReadonlyArray<OrchestrationV2DomainEvent>,
116
+ write: Effect.Effect<A, EventSink.EventSinkV2Error, R>,
117
+ ) => Effect.suspend(() => (matches(events) ? (pending.shift() ?? write) : write));
118
+ return EventSink.EventSinkV2.of({
119
+ ...eventSink,
120
+ write: (input) => failNext(input.events, eventSink.write(input)),
121
+ writeIfRunCurrent: (input) => failNext(input.events, eventSink.writeIfRunCurrent(input)),
122
+ writeIfProviderThreadOwner: (input) =>
123
+ failNext(input.events, eventSink.writeIfProviderThreadOwner(input)),
124
+ });
125
+ }),
126
+ ).pipe(Layer.provide(layerTestEventSink)),
127
+ );
128
129
const makeSharedWorktree = Effect.gen(function* () {
130
const fs = yield* FileSystem.FileSystem;
@@ -2005,7 +2046,7 @@ layer("ProviderEventIngestorV2", (it) => {
2046
}),
2047
);
2048
2008
- it.effect("moves a native subagent's thread to the model its provider reports later", () =>
2049
+ it.effect("moves a native subagent's thread to its later model through a database lock", () =>
2050
Effect.gen(function* () {
2051
const now = yield* DateTime.now;
2052
const eventSink = yield* EventSink.EventSinkV2;
@@ -2124,6 +2165,103 @@ layer("ProviderEventIngestorV2", (it) => {
2165
(yield* projectionStore.getThread(childThreadId)).modelSelection,
2166
reportedSelection,
2167
);
2127
- }),
2168
+ }).pipe(
2169
+ // Fresh: the shared test layer has already built the ingestor this replaces.
2170
+ Effect.provide(
2171
+ Layer.fresh(
2172
+ layerFailingWrites([lockTimeout, lockTimeout], (events) =>
2173
+ events.some((event) => event.type === "thread.model-selection-updated"),
2174
+ ),
2175
+ ),
2176
+ ),
2177
+ ),
2178
);
2179
});
2180
+
2181
+const ingestCompletedProviderTurn = (
2182
+ guard: Pick<
2183
+ Parameters<ProviderEventIngestor.ProviderEventIngestorV2Shape["ingestNormalized"]>[0],
2184
+ "writeIfRunCurrent" | "writeIfProviderThreadOwner"
2185
+ >,
2186
+) =>
2187
+ Effect.gen(function* () {
2188
+ const now = yield* DateTime.now;
2189
+ const eventSink = yield* EventSink.EventSinkV2;
2190
+ const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
2191
+ const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
2192
+ const idAllocator = yield* IdAllocator.IdAllocatorV2;
2193
+ const threadEvent = yield* threadCreatedEvent(now);
2194
+ yield* eventSink.write({ events: [threadEvent] });
2195
+ const providerTurnId = ProviderTurnId.make("provider-turn:locked");
2196
+ const exit = yield* ingestor
2197
+ .ingestNormalized({
2198
+ providerSessionId: yield* idAllocator.allocate.providerSession({
2199
+ providerInstanceId: modelSelection.instanceId,
2200
+ threadId: threadEvent.threadId,
2201
+ }),
2202
+ providerInstanceId: modelSelection.instanceId,
2203
+ threadId: threadEvent.threadId,
2204
+ event: {
2205
+ type: "provider_turn.updated",
2206
+ driver: CODEX_DRIVER,
2207
+ providerTurn: {
2208
+ id: providerTurnId,
2209
+ providerThreadId: ProviderThreadId.make("provider-thread:locked"),
2210
+ nodeId: NodeId.make("node:locked-turn"),
2211
+ runAttemptId: null,
2212
+ nativeTurnRef: null,
2213
+ ordinal: 1,
2214
+ status: "completed",
2215
+ startedAt: now,
2216
+ completedAt: now,
2217
+ },
2218
+ },
2219
+ ...guard,
2220
+ })
2221
+ .pipe(Effect.exit);
2222
+ const projection = yield* projectionStore.getThreadProjection(threadEvent.threadId);
2223
+ return { exit, providerTurnId, persisted: projection.providerTurns.map((turn) => turn.id) };
2224
+ });
2225
+
2226
+// Neither guard matches a stored run, so a guarded write correctly commits nothing.
2227
+const runId = RunId.make("run:locked");
2228
+const activeAttemptId = RunAttemptId.make("attempt:locked");
2229
+it.effect.each([
2230
+ { route: "write", guard: {} },
2231
+ {
2232
+ route: "writeIfRunCurrent",
2233
+ guard: { writeIfRunCurrent: { runId, activeAttemptId, expectedStatus: "running" } },
2234
+ },
2235
+ {
2236
+ route: "writeIfProviderThreadOwner",
2237
+ guard: {
2238
+ writeIfProviderThreadOwner: {
2239
+ providerThreadId: ProviderThreadId.make("provider-thread:locked"),
2240
+ runId,
2241
+ activeAttemptId,
2242
+ expectedLastRunOrdinal: 1,
2243
+ },
2244
+ },
2245
+ },
2246
+] as const)("a provider event $route outlasts a brief database lock", ({ route, guard }) =>
2247
+ Effect.gen(function* () {
2248
+ const { exit, providerTurnId, persisted } = yield* ingestCompletedProviderTurn(guard);
2249
+ assert.isTrue(Exit.isSuccess(exit));
2250
+ assert.deepEqual(persisted, route === "write" ? [providerTurnId] : []);
2251
+ }).pipe(Effect.provide(layerFailingWrites([lockTimeout, lockTimeout]))),
2252
+);
2253
+
2254
+// The write after the queued errors would succeed, so failing proves the retries stopped.
2255
+it.effect.each([
2256
+ { failure: "stays locked", errors: [lockTimeout, lockTimeout, lockTimeout] },
2257
+ { failure: "fails for another reason", errors: [constraintFailure] },
2258
+])("fails a provider event write that $failure", ({ errors }) =>
2259
+ Effect.gen(function* () {
2260
+ const { exit, persisted } = yield* ingestCompletedProviderTurn({});
2261
+ assert.instanceOf(
2262
+ Exit.isFailure(exit) ? Cause.squash(exit.cause) : undefined,
2263
+ ProviderEventIngestor.ProviderEventPublishError,
2264
+ );
2265
+ assert.isEmpty(persisted);
2266
+ }).pipe(Effect.provide(layerFailingWrites(errors))),
2267
+);
apps/server/src/orchestration-v2/ProviderEventIngestor.ts
+13
-4
@@ -27,6 +27,7 @@ import * as DateTime from "effect/DateTime";
27
import * as Effect from "effect/Effect";
28
import * as Layer from "effect/Layer";
29
import * as Schema from "effect/Schema";
30
+import * as SqlError from "effect/sql/SqlError";
31
32
import { AgentScope } from "@t3tools/shared/AgentScope";
33
import { isTemporaryWorktreeBranch } from "@t3tools/shared/git";
@@ -69,6 +70,14 @@ export class ProviderEventPublishError extends Schema.TaggedError<ProviderEventP
70
}
71
}
72
73
+// A write that outlasts busy_timeout at BEGIN IMMEDIATE fails before anything
74
+// commits, and the sink publishes only after a commit, so repeating the whole
75
+// write is safe. Failing it would fail the run over a brief lock.
76
+const isLockTimeout = (error: EventSink.EventSinkV2Error) =>
77
+ SqlError.isSqlError(error.cause) && error.cause.reason._tag === "LockTimeoutError";
78
+const retryLockTimeout = <A, R>(write: Effect.Effect<A, EventSink.EventSinkV2Error, R>) =>
79
+ Effect.retry(write, { while: isLockTimeout, times: 2 });
80
+
81
export const ProviderEventIngestorV2Error = Schema.Union([
82
ProviderEventNormalizeError,
83
ProviderEventPublishError,
@@ -450,7 +459,7 @@ export const layer: Layer.Layer<
459
},
460
occurredAt: now,
461
});
453
- return yield* eventSink.write({ events: [event] });
462
+ return yield* eventSink.write({ events: [event] }).pipe(retryLockTimeout);
463
}),
464
);
465
},
@@ -767,7 +776,7 @@ export const layer: Layer.Layer<
776
...input.writeIfProviderThreadOwner,
777
events,
778
})
770
- .pipe(Effect.mapError(mapWriteError));
779
+ .pipe(retryLockTimeout, Effect.mapError(mapWriteError));
780
return ownerResult.storedEvents;
781
}
782
if (input.writeIfRunCurrent === undefined) {
@@ -777,7 +786,7 @@ export const layer: Layer.Layer<
786
...(input.commandId === undefined ? {} : { commandId: input.commandId }),
787
events,
788
})
780
- .pipe(Effect.mapError(mapWriteError));
789
+ .pipe(retryLockTimeout, Effect.mapError(mapWriteError));
790
}
791
const result = yield* eventSink
792
.writeIfRunCurrent({
@@ -787,7 +796,7 @@ export const layer: Layer.Layer<
796
...input.writeIfRunCurrent,
797
events,
798
})
790
- .pipe(Effect.mapError(mapWriteError));
799
+ .pipe(retryLockTimeout, Effect.mapError(mapWriteError));
800
return result.storedEvents;
801
},
802
(effect, input) =>