The best way to control Claude Code, Codex and more from any computer or phone
fix(server): persist run failures after storage recovers (#19646)
Co-authored-by: GPT-6.1-Sol at high effort <noreply@openai.com> Co-authored-by: Claude Opus 5.5, independent investigation and sanity review <noreply@anthropic.com> Co-authored-by: Julius Marminge <julius0216@outlook.com>
Alb11747 committed
Oct 20, 2026 at 02:05 UTC
801dbf6b84e8f44ef9f94e943ee049aa9f50d8f1
2 files changed
+314
-57
apps/server/src/orchestration-v2/RunExecutionService.test.ts
+245
-4
@@ -36,6 +36,7 @@ import * as Deferred from "effect/Deferred";
36
import * as Effect from "effect/Effect";
37
import * as Fiber from "effect/Fiber";
38
import * as Layer from "effect/Layer";
39
+import * as Logger from "effect/Logger";
40
import * as Option from "effect/Option";
41
import * as Ref from "effect/Ref";
42
import * as Stream from "effect/Stream";
@@ -4104,6 +4105,230 @@ it.effect("keeps completed runs completed when pull request refresh fails", () =
4105
}),
4106
);
4107
4108
+it.effect(
4109
+ "recovers ingestion failure through typed errors and defects with bounded backoff",
4110
+ () => {
4111
+ const warnings: Array<unknown> = [];
4112
+ const logger = Logger.make(({ message }) => {
4113
+ const text = Array.isArray(message) ? message[0] : message;
4114
+ if (text === "orchestration V2 run failure finalization retry") warnings.push(text);
4115
+ });
4116
+ return Effect.gen(function* () {
4117
+ const failIngestion = yield* Deferred.make<void>();
4118
+ const failedWrite = yield* Deferred.make<void>();
4119
+ const writeTimes: Array<number> = [];
4120
+ let storageAvailable = false;
4121
+ let failedAt = yield* DateTime.now;
4122
+ const { finalEvents, observed, written } = yield* captureRootRunTermination({
4123
+ key: "recovery-backoff",
4124
+ shouldFinalizeRun: () => Effect.succeed(true),
4125
+ ingestNormalized: () =>
4126
+ Deferred.await(failIngestion).pipe(Effect.andThen(Effect.die("ingestion failed"))),
4127
+ beforeTerminalWrite: () =>
4128
+ Effect.gen(function* () {
4129
+ writeTimes.push(DateTime.toEpochMillis(yield* DateTime.now));
4130
+ if (storageAvailable) return true;
4131
+ yield* Deferred.succeed(failedWrite, undefined);
4132
+ const error = new EventSink.EventSinkWriteError({ eventCount: 5 });
4133
+ return yield* writeTimes.length % 2 === 1 ? Effect.fail(error) : Effect.die(error);
4134
+ }),
4135
+ afterStart: () =>
4136
+ Effect.gen(function* () {
4137
+ // Separate startup, ingestion failure, and eventual storage recovery.
4138
+ yield* TestClock.adjust("5 seconds");
4139
+ failedAt = yield* DateTime.now;
4140
+ yield* Deferred.succeed(failIngestion, undefined);
4141
+ yield* Deferred.await(failedWrite);
4142
+ yield* TestClock.adjust("999 millis");
4143
+ assert.lengthOf(writeTimes, 1);
4144
+ yield* TestClock.adjust("1 millis");
4145
+ assert.lengthOf(writeTimes, 2);
4146
+ assert.lengthOf(warnings, 1);
4147
+ yield* TestClock.adjust("598999 millis");
4148
+ const offsets = writeTimes.map((time) => time - DateTime.toEpochMillis(failedAt));
4149
+ assert.deepEqual(offsets, [
4150
+ 0,
4151
+ 1000,
4152
+ 3000,
4153
+ 7000,
4154
+ 15000,
4155
+ 31000,
4156
+ ...Array.from({ length: 18 }, (_, index) => 61000 + index * 30000),
4157
+ ]);
4158
+ assert.lengthOf(warnings, 1);
4159
+ yield* TestClock.adjust("1001 millis");
4160
+ assert.lengthOf(warnings, 2);
4161
+ storageAvailable = true;
4162
+ yield* TestClock.adjust("30 seconds");
4163
+ }),
4164
+ });
4165
+ assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
4166
+ assert.lengthOf(warnings, 2);
4167
+ assert.sameMembers(
4168
+ finalEvents.map((event) => event.type),
4169
+ [
4170
+ "run-attempt.updated",
4171
+ "turn-item.updated",
4172
+ "run.updated",
4173
+ "node.updated",
4174
+ "provider-thread.updated",
4175
+ ],
4176
+ );
4177
+ for (const event of finalEvents) {
4178
+ assert.deepEqual(event.occurredAt, failedAt);
4179
+ if (
4180
+ event.type === "run.updated" ||
4181
+ event.type === "run-attempt.updated" ||
4182
+ event.type === "node.updated"
4183
+ ) {
4184
+ assert.equal(event.payload.status, "failed");
4185
+ assert.deepEqual(event.payload.completedAt, failedAt);
4186
+ }
4187
+ if (event.type === "provider-thread.updated")
4188
+ assert.deepEqual(event.payload.updatedAt, failedAt);
4189
+ }
4190
+ assert.lengthOf(written, 1);
4191
+ assert.equal(written[0]?.type, "error");
4192
+ assert.deepEqual(written[0]?.completedAt, failedAt);
4193
+ if (written[0]?.type === "error")
4194
+ assert.equal(written[0].failure.message, "Provider turn failed.");
4195
+ }).pipe(Effect.provide(Logger.layer([logger], { mergeWithExisting: false })));
4196
+ },
4197
+);
4198
+
4199
+it.effect("stops recovery when the atomic gate rejects a superseded attempt", () =>
4200
+ Effect.gen(function* () {
4201
+ const key = "recovery-successor";
4202
+ const ids = backgroundScenarioIds(key);
4203
+ const failedWrite = yield* Deferred.make<void>();
4204
+ let activeAttemptId = ids.attemptId;
4205
+ let writes = 0;
4206
+ const { finalEvents, observed, written } = yield* captureRootRunTermination({
4207
+ key,
4208
+ shouldFinalizeRun: () => Effect.succeed(true),
4209
+ ingestNormalized: () => Effect.die("ingestion failed"),
4210
+ beforeTerminalWrite: (request) =>
4211
+ Effect.gen(function* () {
4212
+ writes++;
4213
+ assert.equal(request.runId, ids.runId);
4214
+ assert.equal(request.activeAttemptId, ids.attemptId);
4215
+ assert.equal(request.expectedStatus, "running");
4216
+ if (writes > 1) return request.activeAttemptId === activeAttemptId;
4217
+ yield* Deferred.succeed(failedWrite, undefined);
4218
+ return yield* new EventSink.EventSinkWriteError({ eventCount: 5 });
4219
+ }),
4220
+ afterStart: () =>
4221
+ Effect.gen(function* () {
4222
+ yield* Deferred.await(failedWrite);
4223
+ activeAttemptId = RunAttemptId.make("attempt:successor");
4224
+ yield* TestClock.adjust("1 second");
4225
+ assert.equal(writes, 2);
4226
+ yield* TestClock.adjust("10 minutes");
4227
+ assert.equal(writes, 2);
4228
+ }),
4229
+ });
4230
+ assert.deepEqual(finalEvents, []);
4231
+ assert.deepEqual(written, []);
4232
+ assert.deepEqual(observed, []);
4233
+ }),
4234
+);
4235
+
4236
+it.effect.each([true, false])(
4237
+ "keeps recovery alive after startup cancels its consumer (startup terminal persisted: %s)",
4238
+ (startupPersisted) =>
4239
+ Effect.gen(function* () {
4240
+ const failedWrite = yield* Deferred.make<void>();
4241
+ let writes = 0;
4242
+ let status: OrchestrationV2Run["status"] = "running";
4243
+ const { observed, finalEvents, written } = yield* captureRootRunTermination({
4244
+ key: `recovery-startup:${startupPersisted}`,
4245
+ shouldFinalizeRun: () => Effect.succeed(true),
4246
+ ingestNormalized: () => Effect.die("ingestion failed"),
4247
+ beforeTerminalWrite: (request) =>
4248
+ Effect.gen(function* () {
4249
+ writes++;
4250
+ if (writes === 1 || (writes === 2 && !startupPersisted)) {
4251
+ yield* Deferred.succeed(failedWrite, undefined);
4252
+ return yield* new EventSink.EventSinkWriteError({ eventCount: 5 });
4253
+ }
4254
+ if (request.expectedStatus !== status) return false;
4255
+ status = "failed";
4256
+ return true;
4257
+ }),
4258
+ startTurn: (input) =>
4259
+ Deferred.await(failedWrite).pipe(
4260
+ Effect.andThen(
4261
+ Effect.fail(
4262
+ new ProviderAdapter.ProviderAdapterTurnStartError({
4263
+ driver,
4264
+ threadId: input.threadId,
4265
+ providerThreadId: input.providerThread.id,
4266
+ runId: input.runId,
4267
+ cause: "provider rejected startup",
4268
+ }),
4269
+ ),
4270
+ ),
4271
+ ),
4272
+ onStartFailure: (error) =>
4273
+ Effect.sync(() => {
4274
+ assert.isFalse(startupPersisted);
4275
+ assert.instanceOf(error, RunExecutionService.RunExecutionStartError);
4276
+ }),
4277
+ afterStart: (consumerClosed) =>
4278
+ Effect.gen(function* () {
4279
+ yield* Deferred.await(consumerClosed);
4280
+ assert.equal(writes, 2);
4281
+ yield* TestClock.adjust("1 second");
4282
+ assert.equal(writes, 3);
4283
+ yield* TestClock.adjust("10 minutes");
4284
+ assert.equal(writes, 3);
4285
+ }),
4286
+ });
4287
+ assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
4288
+ assert.lengthOf(
4289
+ finalEvents.filter(
4290
+ (event) => event.type === "run-attempt.updated" && event.payload.status === "failed",
4291
+ ),
4292
+ 1,
4293
+ );
4294
+ assert.lengthOf(written, 1);
4295
+ assert.equal(written[0]?.type, "error");
4296
+ if (written[0]?.type === "error")
4297
+ assert.include(
4298
+ written[0].failure.message,
4299
+ startupPersisted ? "provider could not start this turn" : "Provider turn failed.",
4300
+ );
4301
+ }),
4302
+);
4303
+
4304
+it.effect("stops pending recovery at service shutdown", () =>
4305
+ Effect.gen(function* () {
4306
+ const failedWrite = yield* Deferred.make<void>();
4307
+ let writes = 0;
4308
+ const exit = yield* captureRootRunTermination({
4309
+ key: "recovery-shutdown",
4310
+ shouldFinalizeRun: () => Effect.succeed(true),
4311
+ ingestNormalized: () => Effect.die("ingestion failed"),
4312
+ beforeTerminalWrite: () =>
4313
+ Effect.gen(function* () {
4314
+ writes++;
4315
+ yield* Deferred.succeed(failedWrite, undefined);
4316
+ return yield* new EventSink.EventSinkWriteError({ eventCount: 5 });
4317
+ }),
4318
+ afterStart: () =>
4319
+ Effect.gen(function* () {
4320
+ yield* Deferred.await(failedWrite);
4321
+ yield* TestClock.adjust("1 second");
4322
+ assert.equal(writes, 2);
4323
+ return yield* Effect.interrupt;
4324
+ }),
4325
+ }).pipe(Effect.exit);
4326
+ assert.isTrue(Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause));
4327
+ yield* TestClock.adjust("10 minutes");
4328
+ assert.equal(writes, 2);
4329
+ }),
4330
+);
4331
+
4332
function captureRootRunTermination(input: {
4333
readonly key: string;
4334
readonly shouldFinalizeRun: () => Effect.Effect<boolean, ProjectionStore.ProjectionStoreV2Error>;
@@ -4117,10 +4342,18 @@ function captureRootRunTermination(input: {
4342
ProviderAdapter.ProviderAdapterV2Error
4343
>;
4344
readonly startTurn?: ProviderAdapter.ProviderAdapterV2SessionRuntime["startTurn"];
4345
+ readonly onStartFailure?: (
4346
+ error: RunExecutionService.RunExecutionServiceV2Error,
4347
+ ) => Effect.Effect<void>;
4348
readonly refreshAfterTurn?: Effect.Effect<void>;
4349
readonly onSubscriptionClose?: Effect.Effect<void>;
4350
readonly shouldStartProviderTurn?: RunExecutionService.RunExecutionServiceV2StartRootRunInput["shouldStartProviderTurn"];
4351
readonly appContext?: ReturnType<McpAppModelContext.McpAppModelContext["Service"]["forThread"]>;
4352
+ readonly ingestNormalized?: ProviderEventIngestor.ProviderEventIngestorV2["Service"]["ingestNormalized"];
4353
+ readonly beforeTerminalWrite?: (
4354
+ request: Parameters<EventSink.EventSinkV2["Service"]["writeIfRunCurrent"]>[0],
4355
+ ) => Effect.Effect<boolean, EventSink.EventSinkV2Error>;
4356
+ readonly afterStart?: (closed: Deferred.Deferred<void>) => Effect.Effect<void>;
4357
}) {
4358
return Effect.gen(function* () {
4359
const ids = backgroundScenarioIds(input.key);
@@ -4181,7 +4414,11 @@ function captureRootRunTermination(input: {
4414
Effect.gen(function* () {
4415
const effects = payload.effects ?? [];
4416
yield* Ref.update(submittedEffects, (current) => [...current, ...effects]);
4184
- if (input.rejectTerminalWrite === true) {
4417
+ if (
4418
+ input.rejectTerminalWrite === true ||
4419
+ (input.beforeTerminalWrite !== undefined &&
4420
+ !(yield* input.beforeTerminalWrite(payload)))
4421
+ ) {
4422
return { committed: false, storedEvents: [] };
4423
}
4424
yield* Ref.update(committedEffects, (current) => [...current, ...effects]);
@@ -4191,7 +4428,7 @@ function captureRootRunTermination(input: {
4428
}),
4429
IdAllocator.layer,
4430
Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
4194
- ingestNormalized: () => Effect.succeed([]),
4431
+ ingestNormalized: input.ingestNormalized ?? (() => Effect.succeed([])),
4432
}),
4433
ServerSettings.layerTest(),
4434
Layer.succeed(RunFinalizationService.RunFinalizationObserver, {
@@ -4207,7 +4444,7 @@ function captureRootRunTermination(input: {
4444
4445
yield* Effect.gen(function* () {
4446
const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
4210
- yield* runExecution.startRootRun({
4447
+ const start = runExecution.startRootRun({
4448
commandId: CommandId.make(`command:${input.key}`),
4449
appThread: { id: ids.threadId } as OrchestrationV2AppThread,
4450
providerSessionId: ProviderSessionId.make(`session:${input.key}`),
@@ -4298,9 +4535,13 @@ function captureRootRunTermination(input: {
4535
},
4536
},
4537
});
4538
+ yield* start.pipe(
4539
+ Effect.catch((error) => input.onStartFailure?.(error) ?? Effect.fail(error)),
4540
+ );
4541
+ yield* input.afterStart?.(ingestionDone) ?? Effect.void;
4542
+ yield* Deferred.await(ingestionDone);
4543
}).pipe(Effect.provide(layerTest));
4544
4303
- yield* Deferred.await(ingestionDone);
4545
return {
4546
written: yield* Ref.get(writtenItems),
4547
finalEvents: yield* Ref.get(finalEvents),
apps/server/src/orchestration-v2/RunExecutionService.ts
+69
-53
@@ -27,11 +27,13 @@ import {
27
import * as Context from "effect/Context";
28
import * as Cause from "effect/Cause";
29
import * as DateTime from "effect/DateTime";
30
+import * as Duration from "effect/Duration";
31
import * as Effect from "effect/Effect";
32
import * as Exit from "effect/Exit";
33
import * as Fiber from "effect/Fiber";
34
import * as Layer from "effect/Layer";
35
import * as Ref from "effect/Ref";
36
+import * as Schedule from "effect/Schedule";
37
import * as Schema from "effect/Schema";
38
import * as Stream from "effect/Stream";
39
@@ -600,6 +602,7 @@ export const layer: Layer.Layer<
602
> = Layer.effect(
603
RunExecutionServiceV2,
604
Effect.gen(function* () {
605
+ const scope = yield* Effect.scope;
606
const checkpointService = yield* CheckpointService.CheckpointServiceV2;
607
const eventSink = yield* EventSink.EventSinkV2;
608
const idAllocator = yield* IdAllocator.IdAllocatorV2;
@@ -620,6 +623,7 @@ export const layer: Layer.Layer<
623
readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect<boolean, never>;
624
readonly openRunOwnedSubagents?: OpenRunOwnedSubagentProjection;
625
readonly terminal: ProviderTerminalEvent;
626
+ readonly completedAt?: DateTime.Utc;
627
readonly failureItemPersisted: boolean;
628
readonly refreshAfterTurn: Effect.Effect<void>;
629
readonly writeIfRunCurrent?: {
@@ -627,7 +631,7 @@ export const layer: Layer.Layer<
631
readonly expectedStatus: OrchestrationV2Run["status"];
632
};
633
}) {
630
- const completedAt = yield* DateTime.now;
634
+ const completedAt = input.completedAt ?? (yield* DateTime.now);
635
const finalizedAttempt: OrchestrationV2RunAttempt | null = {
636
...input.attempt,
637
status: input.terminal.status,
@@ -1424,59 +1428,71 @@ export const layer: Layer.Layer<
1428
Effect.catchCauseIf(
1429
(cause) => !Cause.hasInterruptsOnly(cause),
1430
(cause) =>
1427
- Ref.get(rootRunFinalized).pipe(
1428
- Effect.flatMap((finalized) =>
1429
- Effect.logWarning("orchestration V2 provider event ingestion failed", {
1430
- runId: input.run.id,
1431
- cause,
1432
- }).pipe(
1433
- Effect.andThen(
1434
- finalized
1435
- ? Effect.void
1436
- : Ref.get(latestProviderThread).pipe(
1437
- Effect.flatMap((providerThread) =>
1438
- Ref.get(latestTurnItemOrdinal).pipe(
1439
- Effect.flatMap((latestItemOrdinal) =>
1440
- Ref.get(openRunOwnedSubagents).pipe(
1441
- Effect.flatMap((openSubagents) =>
1442
- writeFinalRunEvents({
1443
- run: input.run,
1444
- rootNode: input.rootNode,
1445
- checkpointScope: input.checkpointScope,
1446
- providerThread,
1447
- attempt: input.attempt,
1448
- writeIfRunCurrent: {
1449
- activeAttemptId: input.attempt.id,
1450
- expectedStatus: "running",
1451
- },
1452
- openRunOwnedSubagents: openSubagents,
1453
- terminal: makeFailedTerminalEvent(
1454
- makeProviderFailure({
1455
- cause: Cause.squash(cause),
1456
- class: "unknown",
1457
- }),
1458
- latestItemOrdinal + 1,
1459
- ),
1460
- failureItemPersisted: false,
1461
- refreshAfterTurn,
1462
- }),
1463
- ),
1464
- ),
1465
- ),
1466
- ),
1467
- ),
1468
- ),
1469
- ),
1470
- Effect.mapError(
1471
- (writeCause) =>
1472
- new RunExecutionIngestError({
1473
- runId: input.run.id,
1474
- cause: { ingest: cause, write: writeCause },
1475
- }),
1476
- ),
1431
+ Effect.gen(function* () {
1432
+ const completedAt = yield* DateTime.now;
1433
+ yield* Effect.logWarning("orchestration V2 provider event ingestion failed", {
1434
+ runId: input.run.id,
1435
+ cause,
1436
+ });
1437
+ if (yield* Ref.get(rootRunFinalized)) return;
1438
+ const providerThread = yield* Ref.get(latestProviderThread);
1439
+ const latestItemOrdinal = yield* Ref.get(latestTurnItemOrdinal);
1440
+ const openSubagents = yield* Ref.get(openRunOwnedSubagents);
1441
+ let nextRetryWarningAt = completedAt;
1442
+ yield* writeFinalRunEvents({
1443
+ run: input.run,
1444
+ rootNode: input.rootNode,
1445
+ checkpointScope: input.checkpointScope,
1446
+ providerThread,
1447
+ attempt: input.attempt,
1448
+ writeIfRunCurrent: {
1449
+ activeAttemptId: input.attempt.id,
1450
+ expectedStatus: "running",
1451
+ },
1452
+ openRunOwnedSubagents: openSubagents,
1453
+ completedAt,
1454
+ terminal: makeFailedTerminalEvent(
1455
+ makeProviderFailure({ cause: Cause.squash(cause), class: "unknown" }),
1456
+ latestItemOrdinal + 1,
1457
),
1478
- ),
1479
- ),
1458
+ failureItemPersisted: false,
1459
+ refreshAfterTurn,
1460
+ }).pipe(
1461
+ // Storage failures can be defects; retry both failure channels.
1462
+ Effect.catchCause((writeCause) =>
1463
+ Effect.gen(function* () {
1464
+ if (Cause.hasInterruptsOnly(writeCause)) {
1465
+ return yield* Effect.failCause(writeCause);
1466
+ }
1467
+ const now = yield* DateTime.now;
1468
+ if (DateTime.isGreaterThanOrEqualTo(now, nextRetryWarningAt)) {
1469
+ yield* Effect.logWarning(
1470
+ "orchestration V2 run failure finalization retry",
1471
+ {
1472
+ runId: input.run.id,
1473
+ attemptId: input.attempt.id,
1474
+ cause: writeCause,
1475
+ },
1476
+ );
1477
+ // Persistent storage failures must not flood the same disk.
1478
+ nextRetryWarningAt = DateTime.add(now, { minutes: 10 });
1479
+ }
1480
+ return yield* Effect.fail(writeCause);
1481
+ }),
1482
+ ),
1483
+ Effect.retry({
1484
+ schedule: Schedule.exponential("1 second").pipe(
1485
+ Schedule.modifyDelay(({ duration }) =>
1486
+ Effect.succeed(Duration.min(duration, Duration.seconds(30))),
1487
+ ),
1488
+ ),
1489
+ }),
1490
+ // Startup may cancel the consumer and fail its own terminal
1491
+ // write. Only service shutdown or the atomic gate ends recovery.
1492
+ Effect.forkIn(scope),
1493
+ Effect.flatMap(Fiber.join),
1494
+ );
1495
+ }),
1496
),
1497
Effect.ensuring(eventSubscription.close),
1498
Effect.forkDetach,