@devries-cameron20 / t3code Mirror · read-only / commits / 97548db62d
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 18:15 UTC

fix(server): retry provider event writes after brief SQLite locks (#24571)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Julius Marminge committed Oct 24, 2026 at 17:17 UTC 97548db62dd68a1859dd15fabb957e3ae746f65f
2 files changed +171 -24
apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts
+158 -20
@@ -17,20 +17,23 @@ import {
17 type OrchestrationV2TurnItem,
18 ProviderDriverKind,
19 ProviderInstanceId,
20 - type ProviderThreadId,
20 + ProviderThreadId,
21 ProviderTurnId,
22 RunAttemptId,
23 RunId,
24 RuntimeRequestId,
25 TurnItemId,
26 } from "@t3tools/contracts";
27 +import * as Cause from "effect/Cause";
28 import * as DateTime from "effect/DateTime";
29 import * as Deferred from "effect/Deferred";
30 import * as Fiber from "effect/Fiber";
31 import * as Effect from "effect/Effect";
32 +import * as Exit from "effect/Exit";
33 import * as FileSystem from "effect/FileSystem";
34 import * as Layer from "effect/Layer";
35 import * as SqlClient from "effect/sql/SqlClient";
36 +import * as SqlError from "effect/sql/SqlError";
37 import * as Stream from "effect/Stream";
38
39 import { toolOutputImages } from "@t3tools/shared/toolOutput";
@@ -65,25 +68,63 @@ const layerGit = GitVcsDriver.layer.pipe(
68 Layer.provideMerge(NodeServices.layer),
69 );
70
68 -const layerTest = Layer.mergeAll(
69 - layerTestDatabase,
70 - layerTestStores,
71 - layerTestEventSink,
72 - IdAllocator.layer,
73 - ThreadCommandExecutor.layer,
74 - layerGit,
75 - ProviderEventIngestor.layer.pipe(
76 - Layer.provide(
77 - Layer.mergeAll(
78 - layerTestStores,
79 - layerTestEventSink,
80 - IdAllocator.layer,
81 - ThreadCommandExecutor.layer,
82 - layerGit,
71 +const makeLayerTest = <E>(ingestorEventSink: Layer.Layer<EventSink.EventSinkV2, E>) =>
72 + Layer.mergeAll(
73 + layerTestDatabase,
74 + layerTestStores,
75 + layerTestEventSink,
76 + IdAllocator.layer,
77 + ThreadCommandExecutor.layer,
78 + layerGit,
79 + ProviderEventIngestor.layer.pipe(
80 + Layer.provide(
81 + Layer.mergeAll(
82 + layerTestStores,
83 + ingestorEventSink,
84 + IdAllocator.layer,
85 + ThreadCommandExecutor.layer,
86 + layerGit,
87 + ),
88 ),
89 ),
85 - ),
86 -);
90 + );
91 +const layerTest = makeLayerTest(layerTestEventSink);
92 +
93 +// EventSink reports a lock that outlasted busy_timeout at BEGIN IMMEDIATE this way.
94 +const lockTimeout = new EventSink.EventSinkWriteError({
95 + eventCount: 1,
96 + cause: new SqlError.SqlError({ reason: new SqlError.LockTimeoutError({ cause: null }) }),
97 +});
98 +const constraintFailure = new EventSink.EventSinkWriteError({
99 + eventCount: 1,
100 + cause: new SqlError.SqlError({ reason: new SqlError.ConstraintError({ cause: null }) }),
101 +});
102 +
103 +/** The ingestor's next sink writes of `events` fail with `errors`, in order; the rest go through. */
104 +const layerFailingWrites = (
105 + errors: ReadonlyArray<EventSink.EventSinkWriteError>,
106 + matches: (events: ReadonlyArray<OrchestrationV2DomainEvent>) => boolean = () => true,
107 +) =>
108 + makeLayerTest(
109 + Layer.effect(
110 + EventSink.EventSinkV2,
111 + Effect.gen(function* () {
112 + const eventSink = yield* EventSink.EventSinkV2;
113 + const pending = [...errors];
114 + const failNext = <A, R>(
115 + events: ReadonlyArray<OrchestrationV2DomainEvent>,
116 + write: Effect.Effect<A, EventSink.EventSinkV2Error, R>,
117 + ) => Effect.suspend(() => (matches(events) ? (pending.shift() ?? write) : write));
118 + return EventSink.EventSinkV2.of({
119 + ...eventSink,
120 + write: (input) => failNext(input.events, eventSink.write(input)),
121 + writeIfRunCurrent: (input) => failNext(input.events, eventSink.writeIfRunCurrent(input)),
122 + writeIfProviderThreadOwner: (input) =>
123 + failNext(input.events, eventSink.writeIfProviderThreadOwner(input)),
124 + });
125 + }),
126 + ).pipe(Layer.provide(layerTestEventSink)),
127 + );
128
129 const makeSharedWorktree = Effect.gen(function* () {
130 const fs = yield* FileSystem.FileSystem;
@@ -2005,7 +2046,7 @@ layer("ProviderEventIngestorV2", (it) => {
2046 }),
2047 );
2048
2008 - it.effect("moves a native subagent's thread to the model its provider reports later", () =>
2049 + it.effect("moves a native subagent's thread to its later model through a database lock", () =>
2050 Effect.gen(function* () {
2051 const now = yield* DateTime.now;
2052 const eventSink = yield* EventSink.EventSinkV2;
@@ -2124,6 +2165,103 @@ layer("ProviderEventIngestorV2", (it) => {
2165 (yield* projectionStore.getThread(childThreadId)).modelSelection,
2166 reportedSelection,
2167 );
2127 - }),
2168 + }).pipe(
2169 + // Fresh: the shared test layer has already built the ingestor this replaces.
2170 + Effect.provide(
2171 + Layer.fresh(
2172 + layerFailingWrites([lockTimeout, lockTimeout], (events) =>
2173 + events.some((event) => event.type === "thread.model-selection-updated"),
2174 + ),
2175 + ),
2176 + ),
2177 + ),
2178 );
2179 });
2180 +
2181 +const ingestCompletedProviderTurn = (
2182 + guard: Pick<
2183 + Parameters<ProviderEventIngestor.ProviderEventIngestorV2Shape["ingestNormalized"]>[0],
2184 + "writeIfRunCurrent" | "writeIfProviderThreadOwner"
2185 + >,
2186 +) =>
2187 + Effect.gen(function* () {
2188 + const now = yield* DateTime.now;
2189 + const eventSink = yield* EventSink.EventSinkV2;
2190 + const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
2191 + const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
2192 + const idAllocator = yield* IdAllocator.IdAllocatorV2;
2193 + const threadEvent = yield* threadCreatedEvent(now);
2194 + yield* eventSink.write({ events: [threadEvent] });
2195 + const providerTurnId = ProviderTurnId.make("provider-turn:locked");
2196 + const exit = yield* ingestor
2197 + .ingestNormalized({
2198 + providerSessionId: yield* idAllocator.allocate.providerSession({
2199 + providerInstanceId: modelSelection.instanceId,
2200 + threadId: threadEvent.threadId,
2201 + }),
2202 + providerInstanceId: modelSelection.instanceId,
2203 + threadId: threadEvent.threadId,
2204 + event: {
2205 + type: "provider_turn.updated",
2206 + driver: CODEX_DRIVER,
2207 + providerTurn: {
2208 + id: providerTurnId,
2209 + providerThreadId: ProviderThreadId.make("provider-thread:locked"),
2210 + nodeId: NodeId.make("node:locked-turn"),
2211 + runAttemptId: null,
2212 + nativeTurnRef: null,
2213 + ordinal: 1,
2214 + status: "completed",
2215 + startedAt: now,
2216 + completedAt: now,
2217 + },
2218 + },
2219 + ...guard,
2220 + })
2221 + .pipe(Effect.exit);
2222 + const projection = yield* projectionStore.getThreadProjection(threadEvent.threadId);
2223 + return { exit, providerTurnId, persisted: projection.providerTurns.map((turn) => turn.id) };
2224 + });
2225 +
2226 +// Neither guard matches a stored run, so a guarded write correctly commits nothing.
2227 +const runId = RunId.make("run:locked");
2228 +const activeAttemptId = RunAttemptId.make("attempt:locked");
2229 +it.effect.each([
2230 + { route: "write", guard: {} },
2231 + {
2232 + route: "writeIfRunCurrent",
2233 + guard: { writeIfRunCurrent: { runId, activeAttemptId, expectedStatus: "running" } },
2234 + },
2235 + {
2236 + route: "writeIfProviderThreadOwner",
2237 + guard: {
2238 + writeIfProviderThreadOwner: {
2239 + providerThreadId: ProviderThreadId.make("provider-thread:locked"),
2240 + runId,
2241 + activeAttemptId,
2242 + expectedLastRunOrdinal: 1,
2243 + },
2244 + },
2245 + },
2246 +] as const)("a provider event $route outlasts a brief database lock", ({ route, guard }) =>
2247 + Effect.gen(function* () {
2248 + const { exit, providerTurnId, persisted } = yield* ingestCompletedProviderTurn(guard);
2249 + assert.isTrue(Exit.isSuccess(exit));
2250 + assert.deepEqual(persisted, route === "write" ? [providerTurnId] : []);
2251 + }).pipe(Effect.provide(layerFailingWrites([lockTimeout, lockTimeout]))),
2252 +);
2253 +
2254 +// The write after the queued errors would succeed, so failing proves the retries stopped.
2255 +it.effect.each([
2256 + { failure: "stays locked", errors: [lockTimeout, lockTimeout, lockTimeout] },
2257 + { failure: "fails for another reason", errors: [constraintFailure] },
2258 +])("fails a provider event write that $failure", ({ errors }) =>
2259 + Effect.gen(function* () {
2260 + const { exit, persisted } = yield* ingestCompletedProviderTurn({});
2261 + assert.instanceOf(
2262 + Exit.isFailure(exit) ? Cause.squash(exit.cause) : undefined,
2263 + ProviderEventIngestor.ProviderEventPublishError,
2264 + );
2265 + assert.isEmpty(persisted);
2266 + }).pipe(Effect.provide(layerFailingWrites(errors))),
2267 +);
apps/server/src/orchestration-v2/ProviderEventIngestor.ts
+13 -4
@@ -27,6 +27,7 @@ import * as DateTime from "effect/DateTime";
27 import * as Effect from "effect/Effect";
28 import * as Layer from "effect/Layer";
29 import * as Schema from "effect/Schema";
30 +import * as SqlError from "effect/sql/SqlError";
31
32 import { AgentScope } from "@t3tools/shared/AgentScope";
33 import { isTemporaryWorktreeBranch } from "@t3tools/shared/git";
@@ -69,6 +70,14 @@ export class ProviderEventPublishError extends Schema.TaggedError<ProviderEventP
70 }
71 }
72
73 +// A write that outlasts busy_timeout at BEGIN IMMEDIATE fails before anything
74 +// commits, and the sink publishes only after a commit, so repeating the whole
75 +// write is safe. Failing it would fail the run over a brief lock.
76 +const isLockTimeout = (error: EventSink.EventSinkV2Error) =>
77 + SqlError.isSqlError(error.cause) && error.cause.reason._tag === "LockTimeoutError";
78 +const retryLockTimeout = <A, R>(write: Effect.Effect<A, EventSink.EventSinkV2Error, R>) =>
79 + Effect.retry(write, { while: isLockTimeout, times: 2 });
80 +
81 export const ProviderEventIngestorV2Error = Schema.Union([
82 ProviderEventNormalizeError,
83 ProviderEventPublishError,
@@ -450,7 +459,7 @@ export const layer: Layer.Layer<
459 },
460 occurredAt: now,
461 });
453 - return yield* eventSink.write({ events: [event] });
462 + return yield* eventSink.write({ events: [event] }).pipe(retryLockTimeout);
463 }),
464 );
465 },
@@ -767,7 +776,7 @@ export const layer: Layer.Layer<
776 ...input.writeIfProviderThreadOwner,
777 events,
778 })
770 - .pipe(Effect.mapError(mapWriteError));
779 + .pipe(retryLockTimeout, Effect.mapError(mapWriteError));
780 return ownerResult.storedEvents;
781 }
782 if (input.writeIfRunCurrent === undefined) {
@@ -777,7 +786,7 @@ export const layer: Layer.Layer<
786 ...(input.commandId === undefined ? {} : { commandId: input.commandId }),
787 events,
788 })
780 - .pipe(Effect.mapError(mapWriteError));
789 + .pipe(retryLockTimeout, Effect.mapError(mapWriteError));
790 }
791 const result = yield* eventSink
792 .writeIfRunCurrent({
@@ -787,7 +796,7 @@ export const layer: Layer.Layer<
796 ...input.writeIfRunCurrent,
797 events,
798 })
790 - .pipe(Effect.mapError(mapWriteError));
799 + .pipe(retryLockTimeout, Effect.mapError(mapWriteError));
800 return result.storedEvents;
801 },
802 (effect, input) =>