@devries-cameron20 / t3code Mirror · read-only / commits / 5ba3a84db4
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 16:00 UTC

fix(server): a locked database no longer leaves a thread stuck in "starting" (#36635)

Co-authored-by: Rene Franke <132335976+ReneFr81@users.noreply.github.com>

Julius Marminge committed Nov 14, 2026 at 02:42 UTC 5ba3a84db457bd9527dbb341fda76dbb24302e02
4 files changed +176 -1
apps/server/src/orchestration-v2/EffectWorker.test.ts
+136
@@ -16,12 +16,16 @@ import {
16 import * as Cause from "effect/Cause";
17 import * as DateTime from "effect/DateTime";
18 import * as Deferred from "effect/Deferred";
19 +import * as Duration from "effect/Duration";
20 import * as Effect from "effect/Effect";
21 import * as Exit from "effect/Exit";
22 +import * as Fiber from "effect/Fiber";
23 import * as Layer from "effect/Layer";
24 import * as Option from "effect/Option";
25 import * as Ref from "effect/Ref";
26 +import * as SqlError from "effect/sql/SqlError";
27 import * as Stream from "effect/Stream";
28 +import * as TestClock from "effect/testing/TestClock";
29
30 import * as CheckpointRollbackService from "./CheckpointRollbackService.ts";
31 import * as CheckpointService from "./CheckpointService.ts";
@@ -396,6 +400,138 @@ it.effect("requeues a claim when a pre-execution worker check fails", () =>
400 }),
401 );
402
403 +it.effect("keeps requeueing a failed claim while the database is locked", () =>
404 + Effect.gen(function* () {
405 + const now = DateTime.formatIso(yield* DateTime.now);
406 + const effectId = "effect:worker-requeue-lock-timeout";
407 + const workerId = "worker-requeue-lock-timeout";
408 + const claimedEffect: EffectOutbox.OrchestrationEffectV2 = {
409 + id: effectId,
410 + commandId: CommandId.make("command:worker-requeue-lock-timeout"),
411 + threadId,
412 + request: { type: "provider-runtime.continue", sourceRunId: runId },
413 + status: "running",
414 + attemptCount: 1,
415 + availableAt: now,
416 + leaseOwner: workerId,
417 + createdAt: now,
418 + updatedAt: now,
419 + completedAt: null,
420 + lastError: null,
421 + };
422 + const lockedWrites = yield* Ref.make(3);
423 + const requeued = yield* Ref.make(false);
424 + const firstLockTimeout = yield* Deferred.make<void>();
425 + const layerOutbox = Layer.mock(EffectOutbox.EffectOutboxV2)({
426 + claimNext: () => Effect.succeedSome(claimedEffect),
427 + get: () => Effect.succeedSome(claimedEffect),
428 + awaitCancellation: () => Effect.never,
429 + clearCancellation: () => Effect.void,
430 + retry: () =>
431 + Effect.gen(function* () {
432 + if ((yield* Ref.getAndUpdate(lockedWrites, (count) => count - 1)) > 0) {
433 + yield* Deferred.succeed(firstLockTimeout, undefined);
434 + return yield* new EffectOutbox.EffectOutboxError({
435 + operation: "retry",
436 + effectId,
437 + cause: new SqlError.SqlError({
438 + reason: new SqlError.LockTimeoutError({ cause: null }),
439 + }),
440 + });
441 + }
442 + yield* Ref.set(requeued, true);
443 + return true;
444 + }),
445 + });
446 + const layerExecutor = Layer.succeed(
447 + EffectWorker.OrchestrationEffectExecutorV2,
448 + EffectWorker.OrchestrationEffectExecutorV2.of({
449 + execute: () =>
450 + Effect.fail(
451 + new EffectWorker.OrchestrationEffectExecutionError({
452 + effectId,
453 + effectType: "provider-runtime.continue",
454 + cause: "simulated database is locked",
455 + }),
456 + ),
457 + }),
458 + );
459 + const layerWorker = EffectWorker.layerWithOptions({ workerId }).pipe(
460 + Layer.provide(Layer.merge(layerOutbox, layerExecutor)),
461 + );
462 +
463 + const workerFiber = yield* EffectWorker.OrchestrationEffectWorkerV2.pipe(
464 + Effect.flatMap((worker) => worker.runOnce()),
465 + Effect.provide(layerWorker),
466 + Effect.forkChild,
467 + );
468 + yield* Deferred.await(firstLockTimeout);
469 + yield* TestClock.adjust(Duration.millis(99));
470 + assert.isFalse(yield* Ref.get(requeued));
471 + yield* TestClock.adjust(Duration.seconds(1));
472 + yield* Effect.exit(Fiber.join(workerFiber));
473 +
474 + assert.isTrue(yield* Ref.get(requeued));
475 + }),
476 +);
477 +
478 +it.effect.each([
479 + { name: "requeue", request: { type: "provider-runtime.continue", sourceRunId: runId } },
480 + { name: "terminalize", request: { type: "provider-turn.start", runId } },
481 +] as const)("interrupts $name settlement retries when the worker shuts down", ({ request }) =>
482 + Effect.gen(function* () {
483 + const claimedEffect: EffectOutbox.OrchestrationEffectV2 = {
484 + ...restartEffect(yield* DateTime.now, { type: "detach" }),
485 + request,
486 + };
487 + const settlementStarted = yield* Deferred.make<void>();
488 + const releaseLock = yield* Ref.make(false);
489 + const lockedWrite = Effect.gen(function* () {
490 + yield* Deferred.succeed(settlementStarted, undefined);
491 + // An in-flight database write can finish its own wait before noticing shutdown.
492 + yield* Effect.sleep(Duration.millis(1));
493 + if (yield* Ref.get(releaseLock)) return true;
494 + return yield* new EffectOutbox.EffectOutboxError({
495 + operation: "settlement",
496 + effectId: claimedEffect.id,
497 + cause: new SqlError.SqlError({ reason: new SqlError.LockTimeoutError({ cause: null }) }),
498 + });
499 + });
500 + const layerOutbox = Layer.mock(EffectOutbox.EffectOutboxV2)({
501 + claimNext: () => Effect.succeedSome(claimedEffect),
502 + get: () => Effect.succeedSome(claimedEffect),
503 + awaitCancellation: () => Effect.never,
504 + clearCancellation: () => Effect.void,
505 + succeed: () => Effect.fail(new EffectOutbox.EffectOutboxError({ operation: "succeed" })),
506 + retry: () => lockedWrite,
507 + fail: () => lockedWrite,
508 + });
509 + const layerExecutor = Layer.succeed(
510 + EffectWorker.OrchestrationEffectExecutorV2,
511 + EffectWorker.OrchestrationEffectExecutorV2.of({ execute: () => Effect.void }),
512 + );
513 + const layerWorker = EffectWorker.layerWithOptions({ workerId: "test-worker" }).pipe(
514 + Layer.provide(Layer.merge(layerOutbox, layerExecutor)),
515 + );
516 +
517 + const daemon = yield* EffectWorker.runDaemonWithOptions({ concurrency: 1 }).pipe(
518 + Effect.provide(layerWorker),
519 + Effect.forkChild,
520 + );
521 + yield* Deferred.await(settlementStarted);
522 + const shutdown = yield* Fiber.interrupt(daemon).pipe(
523 + Effect.timeoutOption(Duration.millis(100)),
524 + Effect.forkChild,
525 + );
526 + yield* TestClock.adjust(Duration.millis(100));
527 + const stopped = yield* Fiber.join(shutdown);
528 + yield* Ref.set(releaseLock, true);
529 + yield* TestClock.adjust(Duration.seconds(1));
530 + yield* Fiber.await(daemon);
531 + assert.isTrue(Option.isSome(stopped));
532 + }),
533 +);
534 +
535 it.effect("arms cancellation before the durable pre-execution check", () =>
536 Effect.gen(function* () {
537 const now = DateTime.formatIso(yield* DateTime.now);
apps/server/src/orchestration-v2/EffectWorker.ts
+14
@@ -9,7 +9,9 @@ import * as Fiber from "effect/Fiber";
9 import * as Layer from "effect/Layer";
10 import * as Metric from "effect/Metric";
11 import * as Option from "effect/Option";
12 +import * as Schedule from "effect/Schedule";
13 import * as Schema from "effect/Schema";
14 +import * as SqlError from "effect/sql/SqlError";
15
16 import {
17 increment,
@@ -667,6 +669,16 @@ export class OrchestrationEffectWorkerError extends Schema.TaggedError<Orchestra
669
670 const isOrchestrationEffectWorkerError = Schema.is(OrchestrationEffectWorkerError);
671
672 +// Only the claiming worker settles its row, and a running row blocks every later effect on its
673 +// thread until the server restarts. onError masks interruption, so retries must restore it.
674 +const isLockTimeout = (error: EffectOutbox.EffectOutboxError) =>
675 + SqlError.isSqlError(error.cause) && error.cause.reason._tag === "LockTimeoutError";
676 +const lockRetrySchedule = Schedule.spaced(Duration.millis(100));
677 +const retryWhileLocked = <A>(settlement: Effect.Effect<A, EffectOutbox.EffectOutboxError>) =>
678 + Effect.interruptible(
679 + Effect.retry(settlement, { schedule: lockRetrySchedule, while: isLockTimeout }),
680 + );
681 +
682 type ClaimFilter = Omit<Parameters<EffectOutbox.EffectOutboxV2Shape["claimNext"]>[0], "workerId">;
683
684 export interface OrchestrationEffectWorkerV2Shape {
@@ -728,6 +740,7 @@ export const layerWithOptions = (
740 delayMs: 0,
741 })
742 .pipe(
743 + retryWhileLocked,
744 Effect.flatMap((requeued) =>
745 requeued
746 ? Effect.logWarning("Requeued effect after unexpected worker failure", {
@@ -759,6 +772,7 @@ export const layerWithOptions = (
772 error: `Worker failed to settle a process-bound effect after execution started: ${Cause.pretty(cause)}`,
773 })
774 .pipe(
775 + retryWhileLocked,
776 Effect.flatMap((failed) =>
777 failed
778 ? Effect.logError("Terminalized process-bound effect after settlement failure", {
packages/shared/src/nodeSqliteClient.test.ts
+19
@@ -240,6 +240,25 @@ it.effect("reuses a worker query after an iterator constraint failure", () =>
240 }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
241 );
242
243 +it.effect("commits on the worker writer after a returning write timed out on a lock", () =>
244 + Effect.gen(function* () {
245 + const { filename, other } = yield* makeTempDatabase;
246 + yield* Effect.gen(function* () {
247 + const sql = yield* SqlClient.SqlClient;
248 + yield* sql`PRAGMA busy_timeout = 0`;
249 + yield* Effect.sync(() => other.exec("BEGIN IMMEDIATE"));
250 + const error = yield* sql`UPDATE counters SET value = 1 WHERE id = 1 RETURNING value`.pipe(
251 + Effect.flip,
252 + );
253 + assert.equal(error.reason._tag, "LockTimeoutError");
254 + yield* Effect.sync(() => other.exec("COMMIT"));
255 +
256 + yield* sql.withTransaction(sql`UPDATE counters SET value = 2 WHERE id = 1`);
257 + assert.deepEqual(yield* sql`SELECT value FROM counters WHERE id = 1`.values, [[2]]);
258 + }).pipe(Effect.provide(SqliteClient.layer({ filename, writerWorker: true })));
259 + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
260 +);
261 +
262 it.effect("keeps another connection from committing between a transaction's read and write", () =>
263 Effect.gen(function* () {
264 const { filename, other } = yield* makeTempDatabase;
packages/shared/src/nodeSqliteClient.ts
+7 -1
@@ -244,6 +244,7 @@ parentPort.on("message", (request) => {
244 }
245 const { id, sql, params, cached, values, raw, safeIntegers, received } = request;
246 let operation = "open";
247 + let iterator;
248 try {
249 if (db === undefined) throw new Error("database connection is unavailable");
250 operation = "prepare";
@@ -255,7 +256,8 @@ parentPort.on("message", (request) => {
256 let rows = [];
257 let bytes = 0;
258 let pendingBatch = false;
258 - for (const row of statement.iterate(...params)) {
259 + iterator = statement.iterate(...params);
260 + for (const row of iterator) {
261 let rowBytes = 0;
262 for (const key in row) {
263 const value = row[key];
@@ -281,6 +283,10 @@ parentPort.on("message", (request) => {
283 parentPort.postMessage({ id, rows: raw ? result : [] });
284 }
285 } catch (error) {
286 + // A step that throws leaves its statement active; an active write blocks every later COMMIT.
287 + try {
288 + iterator?.return();
289 + } catch {}
290 // A failed iterator can report its previous SQLite error when the statement is reused.
291 statements.delete(sql);
292 parentPort.postMessage({ id, error: { operation, ...describe(error) } });