The best way to control Claude Code, Codex and more from any computer or phone
fix(server): a locked database no longer leaves a thread stuck in "starting" (#36635)
Co-authored-by: Rene Franke <132335976+ReneFr81@users.noreply.github.com>
Julius Marminge committed
Nov 14, 2026 at 02:42 UTC
5ba3a84db457bd9527dbb341fda76dbb24302e02
4 files changed
+176
-1
apps/server/src/orchestration-v2/EffectWorker.test.ts
+136
@@ -16,12 +16,16 @@ import {
16
import * as Cause from "effect/Cause";
17
import * as DateTime from "effect/DateTime";
18
import * as Deferred from "effect/Deferred";
19
+import * as Duration from "effect/Duration";
20
import * as Effect from "effect/Effect";
21
import * as Exit from "effect/Exit";
22
+import * as Fiber from "effect/Fiber";
23
import * as Layer from "effect/Layer";
24
import * as Option from "effect/Option";
25
import * as Ref from "effect/Ref";
26
+import * as SqlError from "effect/sql/SqlError";
27
import * as Stream from "effect/Stream";
28
+import * as TestClock from "effect/testing/TestClock";
29
30
import * as CheckpointRollbackService from "./CheckpointRollbackService.ts";
31
import * as CheckpointService from "./CheckpointService.ts";
@@ -396,6 +400,138 @@ it.effect("requeues a claim when a pre-execution worker check fails", () =>
400
}),
401
);
402
403
+it.effect("keeps requeueing a failed claim while the database is locked", () =>
404
+ Effect.gen(function* () {
405
+ const now = DateTime.formatIso(yield* DateTime.now);
406
+ const effectId = "effect:worker-requeue-lock-timeout";
407
+ const workerId = "worker-requeue-lock-timeout";
408
+ const claimedEffect: EffectOutbox.OrchestrationEffectV2 = {
409
+ id: effectId,
410
+ commandId: CommandId.make("command:worker-requeue-lock-timeout"),
411
+ threadId,
412
+ request: { type: "provider-runtime.continue", sourceRunId: runId },
413
+ status: "running",
414
+ attemptCount: 1,
415
+ availableAt: now,
416
+ leaseOwner: workerId,
417
+ createdAt: now,
418
+ updatedAt: now,
419
+ completedAt: null,
420
+ lastError: null,
421
+ };
422
+ const lockedWrites = yield* Ref.make(3);
423
+ const requeued = yield* Ref.make(false);
424
+ const firstLockTimeout = yield* Deferred.make<void>();
425
+ const layerOutbox = Layer.mock(EffectOutbox.EffectOutboxV2)({
426
+ claimNext: () => Effect.succeedSome(claimedEffect),
427
+ get: () => Effect.succeedSome(claimedEffect),
428
+ awaitCancellation: () => Effect.never,
429
+ clearCancellation: () => Effect.void,
430
+ retry: () =>
431
+ Effect.gen(function* () {
432
+ if ((yield* Ref.getAndUpdate(lockedWrites, (count) => count - 1)) > 0) {
433
+ yield* Deferred.succeed(firstLockTimeout, undefined);
434
+ return yield* new EffectOutbox.EffectOutboxError({
435
+ operation: "retry",
436
+ effectId,
437
+ cause: new SqlError.SqlError({
438
+ reason: new SqlError.LockTimeoutError({ cause: null }),
439
+ }),
440
+ });
441
+ }
442
+ yield* Ref.set(requeued, true);
443
+ return true;
444
+ }),
445
+ });
446
+ const layerExecutor = Layer.succeed(
447
+ EffectWorker.OrchestrationEffectExecutorV2,
448
+ EffectWorker.OrchestrationEffectExecutorV2.of({
449
+ execute: () =>
450
+ Effect.fail(
451
+ new EffectWorker.OrchestrationEffectExecutionError({
452
+ effectId,
453
+ effectType: "provider-runtime.continue",
454
+ cause: "simulated database is locked",
455
+ }),
456
+ ),
457
+ }),
458
+ );
459
+ const layerWorker = EffectWorker.layerWithOptions({ workerId }).pipe(
460
+ Layer.provide(Layer.merge(layerOutbox, layerExecutor)),
461
+ );
462
+
463
+ const workerFiber = yield* EffectWorker.OrchestrationEffectWorkerV2.pipe(
464
+ Effect.flatMap((worker) => worker.runOnce()),
465
+ Effect.provide(layerWorker),
466
+ Effect.forkChild,
467
+ );
468
+ yield* Deferred.await(firstLockTimeout);
469
+ yield* TestClock.adjust(Duration.millis(99));
470
+ assert.isFalse(yield* Ref.get(requeued));
471
+ yield* TestClock.adjust(Duration.seconds(1));
472
+ yield* Effect.exit(Fiber.join(workerFiber));
473
+
474
+ assert.isTrue(yield* Ref.get(requeued));
475
+ }),
476
+);
477
+
478
+it.effect.each([
479
+ { name: "requeue", request: { type: "provider-runtime.continue", sourceRunId: runId } },
480
+ { name: "terminalize", request: { type: "provider-turn.start", runId } },
481
+] as const)("interrupts $name settlement retries when the worker shuts down", ({ request }) =>
482
+ Effect.gen(function* () {
483
+ const claimedEffect: EffectOutbox.OrchestrationEffectV2 = {
484
+ ...restartEffect(yield* DateTime.now, { type: "detach" }),
485
+ request,
486
+ };
487
+ const settlementStarted = yield* Deferred.make<void>();
488
+ const releaseLock = yield* Ref.make(false);
489
+ const lockedWrite = Effect.gen(function* () {
490
+ yield* Deferred.succeed(settlementStarted, undefined);
491
+ // An in-flight database write can finish its own wait before noticing shutdown.
492
+ yield* Effect.sleep(Duration.millis(1));
493
+ if (yield* Ref.get(releaseLock)) return true;
494
+ return yield* new EffectOutbox.EffectOutboxError({
495
+ operation: "settlement",
496
+ effectId: claimedEffect.id,
497
+ cause: new SqlError.SqlError({ reason: new SqlError.LockTimeoutError({ cause: null }) }),
498
+ });
499
+ });
500
+ const layerOutbox = Layer.mock(EffectOutbox.EffectOutboxV2)({
501
+ claimNext: () => Effect.succeedSome(claimedEffect),
502
+ get: () => Effect.succeedSome(claimedEffect),
503
+ awaitCancellation: () => Effect.never,
504
+ clearCancellation: () => Effect.void,
505
+ succeed: () => Effect.fail(new EffectOutbox.EffectOutboxError({ operation: "succeed" })),
506
+ retry: () => lockedWrite,
507
+ fail: () => lockedWrite,
508
+ });
509
+ const layerExecutor = Layer.succeed(
510
+ EffectWorker.OrchestrationEffectExecutorV2,
511
+ EffectWorker.OrchestrationEffectExecutorV2.of({ execute: () => Effect.void }),
512
+ );
513
+ const layerWorker = EffectWorker.layerWithOptions({ workerId: "test-worker" }).pipe(
514
+ Layer.provide(Layer.merge(layerOutbox, layerExecutor)),
515
+ );
516
+
517
+ const daemon = yield* EffectWorker.runDaemonWithOptions({ concurrency: 1 }).pipe(
518
+ Effect.provide(layerWorker),
519
+ Effect.forkChild,
520
+ );
521
+ yield* Deferred.await(settlementStarted);
522
+ const shutdown = yield* Fiber.interrupt(daemon).pipe(
523
+ Effect.timeoutOption(Duration.millis(100)),
524
+ Effect.forkChild,
525
+ );
526
+ yield* TestClock.adjust(Duration.millis(100));
527
+ const stopped = yield* Fiber.join(shutdown);
528
+ yield* Ref.set(releaseLock, true);
529
+ yield* TestClock.adjust(Duration.seconds(1));
530
+ yield* Fiber.await(daemon);
531
+ assert.isTrue(Option.isSome(stopped));
532
+ }),
533
+);
534
+
535
it.effect("arms cancellation before the durable pre-execution check", () =>
536
Effect.gen(function* () {
537
const now = DateTime.formatIso(yield* DateTime.now);
apps/server/src/orchestration-v2/EffectWorker.ts
+14
@@ -9,7 +9,9 @@ import * as Fiber from "effect/Fiber";
9
import * as Layer from "effect/Layer";
10
import * as Metric from "effect/Metric";
11
import * as Option from "effect/Option";
12
+import * as Schedule from "effect/Schedule";
13
import * as Schema from "effect/Schema";
14
+import * as SqlError from "effect/sql/SqlError";
15
16
import {
17
increment,
@@ -667,6 +669,16 @@ export class OrchestrationEffectWorkerError extends Schema.TaggedError<Orchestra
669
670
const isOrchestrationEffectWorkerError = Schema.is(OrchestrationEffectWorkerError);
671
672
+// Only the claiming worker settles its row, and a running row blocks every later effect on its
673
+// thread until the server restarts. onError masks interruption, so retries must restore it.
674
+const isLockTimeout = (error: EffectOutbox.EffectOutboxError) =>
675
+ SqlError.isSqlError(error.cause) && error.cause.reason._tag === "LockTimeoutError";
676
+const lockRetrySchedule = Schedule.spaced(Duration.millis(100));
677
+const retryWhileLocked = <A>(settlement: Effect.Effect<A, EffectOutbox.EffectOutboxError>) =>
678
+ Effect.interruptible(
679
+ Effect.retry(settlement, { schedule: lockRetrySchedule, while: isLockTimeout }),
680
+ );
681
+
682
type ClaimFilter = Omit<Parameters<EffectOutbox.EffectOutboxV2Shape["claimNext"]>[0], "workerId">;
683
684
export interface OrchestrationEffectWorkerV2Shape {
@@ -728,6 +740,7 @@ export const layerWithOptions = (
740
delayMs: 0,
741
})
742
.pipe(
743
+ retryWhileLocked,
744
Effect.flatMap((requeued) =>
745
requeued
746
? Effect.logWarning("Requeued effect after unexpected worker failure", {
@@ -759,6 +772,7 @@ export const layerWithOptions = (
772
error: `Worker failed to settle a process-bound effect after execution started: ${Cause.pretty(cause)}`,
773
})
774
.pipe(
775
+ retryWhileLocked,
776
Effect.flatMap((failed) =>
777
failed
778
? Effect.logError("Terminalized process-bound effect after settlement failure", {
packages/shared/src/nodeSqliteClient.test.ts
+19
@@ -240,6 +240,25 @@ it.effect("reuses a worker query after an iterator constraint failure", () =>
240
}).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
241
);
242
243
+it.effect("commits on the worker writer after a returning write timed out on a lock", () =>
244
+ Effect.gen(function* () {
245
+ const { filename, other } = yield* makeTempDatabase;
246
+ yield* Effect.gen(function* () {
247
+ const sql = yield* SqlClient.SqlClient;
248
+ yield* sql`PRAGMA busy_timeout = 0`;
249
+ yield* Effect.sync(() => other.exec("BEGIN IMMEDIATE"));
250
+ const error = yield* sql`UPDATE counters SET value = 1 WHERE id = 1 RETURNING value`.pipe(
251
+ Effect.flip,
252
+ );
253
+ assert.equal(error.reason._tag, "LockTimeoutError");
254
+ yield* Effect.sync(() => other.exec("COMMIT"));
255
+
256
+ yield* sql.withTransaction(sql`UPDATE counters SET value = 2 WHERE id = 1`);
257
+ assert.deepEqual(yield* sql`SELECT value FROM counters WHERE id = 1`.values, [[2]]);
258
+ }).pipe(Effect.provide(SqliteClient.layer({ filename, writerWorker: true })));
259
+ }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
260
+);
261
+
262
it.effect("keeps another connection from committing between a transaction's read and write", () =>
263
Effect.gen(function* () {
264
const { filename, other } = yield* makeTempDatabase;
packages/shared/src/nodeSqliteClient.ts
+7
-1
@@ -244,6 +244,7 @@ parentPort.on("message", (request) => {
244
}
245
const { id, sql, params, cached, values, raw, safeIntegers, received } = request;
246
let operation = "open";
247
+ let iterator;
248
try {
249
if (db === undefined) throw new Error("database connection is unavailable");
250
operation = "prepare";
@@ -255,7 +256,8 @@ parentPort.on("message", (request) => {
256
let rows = [];
257
let bytes = 0;
258
let pendingBatch = false;
258
- for (const row of statement.iterate(...params)) {
259
+ iterator = statement.iterate(...params);
260
+ for (const row of iterator) {
261
let rowBytes = 0;
262
for (const key in row) {
263
const value = row[key];
@@ -281,6 +283,10 @@ parentPort.on("message", (request) => {
283
parentPort.postMessage({ id, rows: raw ? result : [] });
284
}
285
} catch (error) {
286
+ // A step that throws leaves its statement active; an active write blocks every later COMMIT.
287
+ try {
288
+ iterator?.return();
289
+ } catch {}
290
// A failed iterator can report its previous SQLite error when the statement is reused.
291
statements.delete(sql);
292
parentPort.postMessage({ id, error: { operation, ...describe(error) } });