@devries-cameron20 / t3code-1 / commits / 801dbf6b84
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): persist run failures after storage recovers (#19646)

Co-authored-by: GPT-6.1-Sol at high effort <noreply@openai.com> Co-authored-by: Claude Opus 5.5, independent investigation and sanity review <noreply@anthropic.com> Co-authored-by: Julius Marminge <julius0216@outlook.com>

Alb11747 committed Oct 20, 2026 at 02:05 UTC 801dbf6b84e8f44ef9f94e943ee049aa9f50d8f1
2 files changed +314 -57
apps/server/src/orchestration-v2/RunExecutionService.test.ts
+245 -4
@@ -36,6 +36,7 @@ import * as Deferred from "effect/Deferred";
36 import * as Effect from "effect/Effect";
37 import * as Fiber from "effect/Fiber";
38 import * as Layer from "effect/Layer";
39 +import * as Logger from "effect/Logger";
40 import * as Option from "effect/Option";
41 import * as Ref from "effect/Ref";
42 import * as Stream from "effect/Stream";
@@ -4104,6 +4105,230 @@ it.effect("keeps completed runs completed when pull request refresh fails", () =
4105 }),
4106 );
4107
4108 +it.effect(
4109 + "recovers ingestion failure through typed errors and defects with bounded backoff",
4110 + () => {
4111 + const warnings: Array<unknown> = [];
4112 + const logger = Logger.make(({ message }) => {
4113 + const text = Array.isArray(message) ? message[0] : message;
4114 + if (text === "orchestration V2 run failure finalization retry") warnings.push(text);
4115 + });
4116 + return Effect.gen(function* () {
4117 + const failIngestion = yield* Deferred.make<void>();
4118 + const failedWrite = yield* Deferred.make<void>();
4119 + const writeTimes: Array<number> = [];
4120 + let storageAvailable = false;
4121 + let failedAt = yield* DateTime.now;
4122 + const { finalEvents, observed, written } = yield* captureRootRunTermination({
4123 + key: "recovery-backoff",
4124 + shouldFinalizeRun: () => Effect.succeed(true),
4125 + ingestNormalized: () =>
4126 + Deferred.await(failIngestion).pipe(Effect.andThen(Effect.die("ingestion failed"))),
4127 + beforeTerminalWrite: () =>
4128 + Effect.gen(function* () {
4129 + writeTimes.push(DateTime.toEpochMillis(yield* DateTime.now));
4130 + if (storageAvailable) return true;
4131 + yield* Deferred.succeed(failedWrite, undefined);
4132 + const error = new EventSink.EventSinkWriteError({ eventCount: 5 });
4133 + return yield* writeTimes.length % 2 === 1 ? Effect.fail(error) : Effect.die(error);
4134 + }),
4135 + afterStart: () =>
4136 + Effect.gen(function* () {
4137 + // Separate startup, ingestion failure, and eventual storage recovery.
4138 + yield* TestClock.adjust("5 seconds");
4139 + failedAt = yield* DateTime.now;
4140 + yield* Deferred.succeed(failIngestion, undefined);
4141 + yield* Deferred.await(failedWrite);
4142 + yield* TestClock.adjust("999 millis");
4143 + assert.lengthOf(writeTimes, 1);
4144 + yield* TestClock.adjust("1 millis");
4145 + assert.lengthOf(writeTimes, 2);
4146 + assert.lengthOf(warnings, 1);
4147 + yield* TestClock.adjust("598999 millis");
4148 + const offsets = writeTimes.map((time) => time - DateTime.toEpochMillis(failedAt));
4149 + assert.deepEqual(offsets, [
4150 + 0,
4151 + 1000,
4152 + 3000,
4153 + 7000,
4154 + 15000,
4155 + 31000,
4156 + ...Array.from({ length: 18 }, (_, index) => 61000 + index * 30000),
4157 + ]);
4158 + assert.lengthOf(warnings, 1);
4159 + yield* TestClock.adjust("1001 millis");
4160 + assert.lengthOf(warnings, 2);
4161 + storageAvailable = true;
4162 + yield* TestClock.adjust("30 seconds");
4163 + }),
4164 + });
4165 + assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
4166 + assert.lengthOf(warnings, 2);
4167 + assert.sameMembers(
4168 + finalEvents.map((event) => event.type),
4169 + [
4170 + "run-attempt.updated",
4171 + "turn-item.updated",
4172 + "run.updated",
4173 + "node.updated",
4174 + "provider-thread.updated",
4175 + ],
4176 + );
4177 + for (const event of finalEvents) {
4178 + assert.deepEqual(event.occurredAt, failedAt);
4179 + if (
4180 + event.type === "run.updated" ||
4181 + event.type === "run-attempt.updated" ||
4182 + event.type === "node.updated"
4183 + ) {
4184 + assert.equal(event.payload.status, "failed");
4185 + assert.deepEqual(event.payload.completedAt, failedAt);
4186 + }
4187 + if (event.type === "provider-thread.updated")
4188 + assert.deepEqual(event.payload.updatedAt, failedAt);
4189 + }
4190 + assert.lengthOf(written, 1);
4191 + assert.equal(written[0]?.type, "error");
4192 + assert.deepEqual(written[0]?.completedAt, failedAt);
4193 + if (written[0]?.type === "error")
4194 + assert.equal(written[0].failure.message, "Provider turn failed.");
4195 + }).pipe(Effect.provide(Logger.layer([logger], { mergeWithExisting: false })));
4196 + },
4197 +);
4198 +
4199 +it.effect("stops recovery when the atomic gate rejects a superseded attempt", () =>
4200 + Effect.gen(function* () {
4201 + const key = "recovery-successor";
4202 + const ids = backgroundScenarioIds(key);
4203 + const failedWrite = yield* Deferred.make<void>();
4204 + let activeAttemptId = ids.attemptId;
4205 + let writes = 0;
4206 + const { finalEvents, observed, written } = yield* captureRootRunTermination({
4207 + key,
4208 + shouldFinalizeRun: () => Effect.succeed(true),
4209 + ingestNormalized: () => Effect.die("ingestion failed"),
4210 + beforeTerminalWrite: (request) =>
4211 + Effect.gen(function* () {
4212 + writes++;
4213 + assert.equal(request.runId, ids.runId);
4214 + assert.equal(request.activeAttemptId, ids.attemptId);
4215 + assert.equal(request.expectedStatus, "running");
4216 + if (writes > 1) return request.activeAttemptId === activeAttemptId;
4217 + yield* Deferred.succeed(failedWrite, undefined);
4218 + return yield* new EventSink.EventSinkWriteError({ eventCount: 5 });
4219 + }),
4220 + afterStart: () =>
4221 + Effect.gen(function* () {
4222 + yield* Deferred.await(failedWrite);
4223 + activeAttemptId = RunAttemptId.make("attempt:successor");
4224 + yield* TestClock.adjust("1 second");
4225 + assert.equal(writes, 2);
4226 + yield* TestClock.adjust("10 minutes");
4227 + assert.equal(writes, 2);
4228 + }),
4229 + });
4230 + assert.deepEqual(finalEvents, []);
4231 + assert.deepEqual(written, []);
4232 + assert.deepEqual(observed, []);
4233 + }),
4234 +);
4235 +
4236 +it.effect.each([true, false])(
4237 + "keeps recovery alive after startup cancels its consumer (startup terminal persisted: %s)",
4238 + (startupPersisted) =>
4239 + Effect.gen(function* () {
4240 + const failedWrite = yield* Deferred.make<void>();
4241 + let writes = 0;
4242 + let status: OrchestrationV2Run["status"] = "running";
4243 + const { observed, finalEvents, written } = yield* captureRootRunTermination({
4244 + key: `recovery-startup:${startupPersisted}`,
4245 + shouldFinalizeRun: () => Effect.succeed(true),
4246 + ingestNormalized: () => Effect.die("ingestion failed"),
4247 + beforeTerminalWrite: (request) =>
4248 + Effect.gen(function* () {
4249 + writes++;
4250 + if (writes === 1 || (writes === 2 && !startupPersisted)) {
4251 + yield* Deferred.succeed(failedWrite, undefined);
4252 + return yield* new EventSink.EventSinkWriteError({ eventCount: 5 });
4253 + }
4254 + if (request.expectedStatus !== status) return false;
4255 + status = "failed";
4256 + return true;
4257 + }),
4258 + startTurn: (input) =>
4259 + Deferred.await(failedWrite).pipe(
4260 + Effect.andThen(
4261 + Effect.fail(
4262 + new ProviderAdapter.ProviderAdapterTurnStartError({
4263 + driver,
4264 + threadId: input.threadId,
4265 + providerThreadId: input.providerThread.id,
4266 + runId: input.runId,
4267 + cause: "provider rejected startup",
4268 + }),
4269 + ),
4270 + ),
4271 + ),
4272 + onStartFailure: (error) =>
4273 + Effect.sync(() => {
4274 + assert.isFalse(startupPersisted);
4275 + assert.instanceOf(error, RunExecutionService.RunExecutionStartError);
4276 + }),
4277 + afterStart: (consumerClosed) =>
4278 + Effect.gen(function* () {
4279 + yield* Deferred.await(consumerClosed);
4280 + assert.equal(writes, 2);
4281 + yield* TestClock.adjust("1 second");
4282 + assert.equal(writes, 3);
4283 + yield* TestClock.adjust("10 minutes");
4284 + assert.equal(writes, 3);
4285 + }),
4286 + });
4287 + assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
4288 + assert.lengthOf(
4289 + finalEvents.filter(
4290 + (event) => event.type === "run-attempt.updated" && event.payload.status === "failed",
4291 + ),
4292 + 1,
4293 + );
4294 + assert.lengthOf(written, 1);
4295 + assert.equal(written[0]?.type, "error");
4296 + if (written[0]?.type === "error")
4297 + assert.include(
4298 + written[0].failure.message,
4299 + startupPersisted ? "provider could not start this turn" : "Provider turn failed.",
4300 + );
4301 + }),
4302 +);
4303 +
4304 +it.effect("stops pending recovery at service shutdown", () =>
4305 + Effect.gen(function* () {
4306 + const failedWrite = yield* Deferred.make<void>();
4307 + let writes = 0;
4308 + const exit = yield* captureRootRunTermination({
4309 + key: "recovery-shutdown",
4310 + shouldFinalizeRun: () => Effect.succeed(true),
4311 + ingestNormalized: () => Effect.die("ingestion failed"),
4312 + beforeTerminalWrite: () =>
4313 + Effect.gen(function* () {
4314 + writes++;
4315 + yield* Deferred.succeed(failedWrite, undefined);
4316 + return yield* new EventSink.EventSinkWriteError({ eventCount: 5 });
4317 + }),
4318 + afterStart: () =>
4319 + Effect.gen(function* () {
4320 + yield* Deferred.await(failedWrite);
4321 + yield* TestClock.adjust("1 second");
4322 + assert.equal(writes, 2);
4323 + return yield* Effect.interrupt;
4324 + }),
4325 + }).pipe(Effect.exit);
4326 + assert.isTrue(Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause));
4327 + yield* TestClock.adjust("10 minutes");
4328 + assert.equal(writes, 2);
4329 + }),
4330 +);
4331 +
4332 function captureRootRunTermination(input: {
4333 readonly key: string;
4334 readonly shouldFinalizeRun: () => Effect.Effect<boolean, ProjectionStore.ProjectionStoreV2Error>;
@@ -4117,10 +4342,18 @@ function captureRootRunTermination(input: {
4342 ProviderAdapter.ProviderAdapterV2Error
4343 >;
4344 readonly startTurn?: ProviderAdapter.ProviderAdapterV2SessionRuntime["startTurn"];
4345 + readonly onStartFailure?: (
4346 + error: RunExecutionService.RunExecutionServiceV2Error,
4347 + ) => Effect.Effect<void>;
4348 readonly refreshAfterTurn?: Effect.Effect<void>;
4349 readonly onSubscriptionClose?: Effect.Effect<void>;
4350 readonly shouldStartProviderTurn?: RunExecutionService.RunExecutionServiceV2StartRootRunInput["shouldStartProviderTurn"];
4351 readonly appContext?: ReturnType<McpAppModelContext.McpAppModelContext["Service"]["forThread"]>;
4352 + readonly ingestNormalized?: ProviderEventIngestor.ProviderEventIngestorV2["Service"]["ingestNormalized"];
4353 + readonly beforeTerminalWrite?: (
4354 + request: Parameters<EventSink.EventSinkV2["Service"]["writeIfRunCurrent"]>[0],
4355 + ) => Effect.Effect<boolean, EventSink.EventSinkV2Error>;
4356 + readonly afterStart?: (closed: Deferred.Deferred<void>) => Effect.Effect<void>;
4357 }) {
4358 return Effect.gen(function* () {
4359 const ids = backgroundScenarioIds(input.key);
@@ -4181,7 +4414,11 @@ function captureRootRunTermination(input: {
4414 Effect.gen(function* () {
4415 const effects = payload.effects ?? [];
4416 yield* Ref.update(submittedEffects, (current) => [...current, ...effects]);
4184 - if (input.rejectTerminalWrite === true) {
4417 + if (
4418 + input.rejectTerminalWrite === true ||
4419 + (input.beforeTerminalWrite !== undefined &&
4420 + !(yield* input.beforeTerminalWrite(payload)))
4421 + ) {
4422 return { committed: false, storedEvents: [] };
4423 }
4424 yield* Ref.update(committedEffects, (current) => [...current, ...effects]);
@@ -4191,7 +4428,7 @@ function captureRootRunTermination(input: {
4428 }),
4429 IdAllocator.layer,
4430 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
4194 - ingestNormalized: () => Effect.succeed([]),
4431 + ingestNormalized: input.ingestNormalized ?? (() => Effect.succeed([])),
4432 }),
4433 ServerSettings.layerTest(),
4434 Layer.succeed(RunFinalizationService.RunFinalizationObserver, {
@@ -4207,7 +4444,7 @@ function captureRootRunTermination(input: {
4444
4445 yield* Effect.gen(function* () {
4446 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
4210 - yield* runExecution.startRootRun({
4447 + const start = runExecution.startRootRun({
4448 commandId: CommandId.make(`command:${input.key}`),
4449 appThread: { id: ids.threadId } as OrchestrationV2AppThread,
4450 providerSessionId: ProviderSessionId.make(`session:${input.key}`),
@@ -4298,9 +4535,13 @@ function captureRootRunTermination(input: {
4535 },
4536 },
4537 });
4538 + yield* start.pipe(
4539 + Effect.catch((error) => input.onStartFailure?.(error) ?? Effect.fail(error)),
4540 + );
4541 + yield* input.afterStart?.(ingestionDone) ?? Effect.void;
4542 + yield* Deferred.await(ingestionDone);
4543 }).pipe(Effect.provide(layerTest));
4544
4303 - yield* Deferred.await(ingestionDone);
4545 return {
4546 written: yield* Ref.get(writtenItems),
4547 finalEvents: yield* Ref.get(finalEvents),
apps/server/src/orchestration-v2/RunExecutionService.ts
+69 -53
@@ -27,11 +27,13 @@ import {
27 import * as Context from "effect/Context";
28 import * as Cause from "effect/Cause";
29 import * as DateTime from "effect/DateTime";
30 +import * as Duration from "effect/Duration";
31 import * as Effect from "effect/Effect";
32 import * as Exit from "effect/Exit";
33 import * as Fiber from "effect/Fiber";
34 import * as Layer from "effect/Layer";
35 import * as Ref from "effect/Ref";
36 +import * as Schedule from "effect/Schedule";
37 import * as Schema from "effect/Schema";
38 import * as Stream from "effect/Stream";
39
@@ -600,6 +602,7 @@ export const layer: Layer.Layer<
602 > = Layer.effect(
603 RunExecutionServiceV2,
604 Effect.gen(function* () {
605 + const scope = yield* Effect.scope;
606 const checkpointService = yield* CheckpointService.CheckpointServiceV2;
607 const eventSink = yield* EventSink.EventSinkV2;
608 const idAllocator = yield* IdAllocator.IdAllocatorV2;
@@ -620,6 +623,7 @@ export const layer: Layer.Layer<
623 readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect<boolean, never>;
624 readonly openRunOwnedSubagents?: OpenRunOwnedSubagentProjection;
625 readonly terminal: ProviderTerminalEvent;
626 + readonly completedAt?: DateTime.Utc;
627 readonly failureItemPersisted: boolean;
628 readonly refreshAfterTurn: Effect.Effect<void>;
629 readonly writeIfRunCurrent?: {
@@ -627,7 +631,7 @@ export const layer: Layer.Layer<
631 readonly expectedStatus: OrchestrationV2Run["status"];
632 };
633 }) {
630 - const completedAt = yield* DateTime.now;
634 + const completedAt = input.completedAt ?? (yield* DateTime.now);
635 const finalizedAttempt: OrchestrationV2RunAttempt | null = {
636 ...input.attempt,
637 status: input.terminal.status,
@@ -1424,59 +1428,71 @@ export const layer: Layer.Layer<
1428 Effect.catchCauseIf(
1429 (cause) => !Cause.hasInterruptsOnly(cause),
1430 (cause) =>
1427 - Ref.get(rootRunFinalized).pipe(
1428 - Effect.flatMap((finalized) =>
1429 - Effect.logWarning("orchestration V2 provider event ingestion failed", {
1430 - runId: input.run.id,
1431 - cause,
1432 - }).pipe(
1433 - Effect.andThen(
1434 - finalized
1435 - ? Effect.void
1436 - : Ref.get(latestProviderThread).pipe(
1437 - Effect.flatMap((providerThread) =>
1438 - Ref.get(latestTurnItemOrdinal).pipe(
1439 - Effect.flatMap((latestItemOrdinal) =>
1440 - Ref.get(openRunOwnedSubagents).pipe(
1441 - Effect.flatMap((openSubagents) =>
1442 - writeFinalRunEvents({
1443 - run: input.run,
1444 - rootNode: input.rootNode,
1445 - checkpointScope: input.checkpointScope,
1446 - providerThread,
1447 - attempt: input.attempt,
1448 - writeIfRunCurrent: {
1449 - activeAttemptId: input.attempt.id,
1450 - expectedStatus: "running",
1451 - },
1452 - openRunOwnedSubagents: openSubagents,
1453 - terminal: makeFailedTerminalEvent(
1454 - makeProviderFailure({
1455 - cause: Cause.squash(cause),
1456 - class: "unknown",
1457 - }),
1458 - latestItemOrdinal + 1,
1459 - ),
1460 - failureItemPersisted: false,
1461 - refreshAfterTurn,
1462 - }),
1463 - ),
1464 - ),
1465 - ),
1466 - ),
1467 - ),
1468 - ),
1469 - ),
1470 - Effect.mapError(
1471 - (writeCause) =>
1472 - new RunExecutionIngestError({
1473 - runId: input.run.id,
1474 - cause: { ingest: cause, write: writeCause },
1475 - }),
1476 - ),
1431 + Effect.gen(function* () {
1432 + const completedAt = yield* DateTime.now;
1433 + yield* Effect.logWarning("orchestration V2 provider event ingestion failed", {
1434 + runId: input.run.id,
1435 + cause,
1436 + });
1437 + if (yield* Ref.get(rootRunFinalized)) return;
1438 + const providerThread = yield* Ref.get(latestProviderThread);
1439 + const latestItemOrdinal = yield* Ref.get(latestTurnItemOrdinal);
1440 + const openSubagents = yield* Ref.get(openRunOwnedSubagents);
1441 + let nextRetryWarningAt = completedAt;
1442 + yield* writeFinalRunEvents({
1443 + run: input.run,
1444 + rootNode: input.rootNode,
1445 + checkpointScope: input.checkpointScope,
1446 + providerThread,
1447 + attempt: input.attempt,
1448 + writeIfRunCurrent: {
1449 + activeAttemptId: input.attempt.id,
1450 + expectedStatus: "running",
1451 + },
1452 + openRunOwnedSubagents: openSubagents,
1453 + completedAt,
1454 + terminal: makeFailedTerminalEvent(
1455 + makeProviderFailure({ cause: Cause.squash(cause), class: "unknown" }),
1456 + latestItemOrdinal + 1,
1457 ),
1478 - ),
1479 - ),
1458 + failureItemPersisted: false,
1459 + refreshAfterTurn,
1460 + }).pipe(
1461 + // Storage failures can be defects; retry both failure channels.
1462 + Effect.catchCause((writeCause) =>
1463 + Effect.gen(function* () {
1464 + if (Cause.hasInterruptsOnly(writeCause)) {
1465 + return yield* Effect.failCause(writeCause);
1466 + }
1467 + const now = yield* DateTime.now;
1468 + if (DateTime.isGreaterThanOrEqualTo(now, nextRetryWarningAt)) {
1469 + yield* Effect.logWarning(
1470 + "orchestration V2 run failure finalization retry",
1471 + {
1472 + runId: input.run.id,
1473 + attemptId: input.attempt.id,
1474 + cause: writeCause,
1475 + },
1476 + );
1477 + // Persistent storage failures must not flood the same disk.
1478 + nextRetryWarningAt = DateTime.add(now, { minutes: 10 });
1479 + }
1480 + return yield* Effect.fail(writeCause);
1481 + }),
1482 + ),
1483 + Effect.retry({
1484 + schedule: Schedule.exponential("1 second").pipe(
1485 + Schedule.modifyDelay(({ duration }) =>
1486 + Effect.succeed(Duration.min(duration, Duration.seconds(30))),
1487 + ),
1488 + ),
1489 + }),
1490 + // Startup may cancel the consumer and fail its own terminal
1491 + // write. Only service shutdown or the atomic gate ends recovery.
1492 + Effect.forkIn(scope),
1493 + Effect.flatMap(Fiber.join),
1494 + );
1495 + }),
1496 ),
1497 Effect.ensuring(eventSubscription.close),
1498 Effect.forkDetach,