The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 13:30 UTC
fix(server): a failed turn start no longer leaves the thread stuck starting (#14424)
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: SAPHID <saphid@users.noreply.github.com> Co-authored-by: Julius Marminge <julius0216@outlook.com>
Alex Southwell committed
Nov 15, 2026 at 07:58 UTC
c76ba848ab5d83e1babfd1db731bd5ce92ff4312
4 files changed
+822
-166
apps/server/src/orchestration-v2/EffectWorker.test.ts
+1
@@ -126,6 +126,7 @@ function layerExecutorFor(input: {
126
Layer.succeed(
127
ProviderTurnStartService.ProviderTurnStartServiceV2,
128
ProviderTurnStartService.ProviderTurnStartServiceV2.of({
129
+ failStartingRun: () => Effect.void,
130
start: () =>
131
Effect.gen(function* () {
132
yield* record("start");
apps/server/src/orchestration-v2/EffectWorker.ts
+67
-19
@@ -1,4 +1,4 @@
1
-import { CommandId } from "@t3tools/contracts";
1
+import { CommandId, type RunId } from "@t3tools/contracts";
2
import * as Cause from "effect/Cause";
3
import * as Context from "effect/Context";
4
import * as DateTime from "effect/DateTime";
@@ -44,6 +44,8 @@ export class OrchestrationEffectExecutionError extends Schema.TaggedError<Orches
44
},
45
) {}
46
47
+const isOrchestrationEffectExecutionError = Schema.is(OrchestrationEffectExecutionError);
48
+
49
/**
50
* Do not apply this to `provider-turn.restart`: that compound effect also runs
51
* detach and start. Swallowing a start failure that happens to mention
@@ -62,6 +64,19 @@ function isNonRetryableProviderTurnControlFailure(effectType: string, errorText:
64
);
65
}
66
67
+// Keep an owner for starting runs when persistence or restart cleanup fails.
68
+const ownsStartingRun = (effectType: EffectOutbox.OrchestrationEffectV2["request"]["type"]) =>
69
+ effectType === "provider-turn.start" || effectType === "provider-turn.restart";
70
+
71
+const cannotSettleRun = (cause: Cause.Cause<unknown>) =>
72
+ Cause.findErrorOption(cause).pipe(
73
+ Option.exists(
74
+ (error) =>
75
+ isOrchestrationEffectExecutionError(error) &&
76
+ ProviderTurnStartService.isProviderTurnStartRunStateMissingError(error.cause),
77
+ ),
78
+ );
79
+
80
export interface OrchestrationEffectExecutorV2Shape {
81
readonly execute: (
82
effect: EffectOutbox.OrchestrationEffectV2,
@@ -102,6 +117,43 @@ export const layerExecutor: Layer.Layer<
117
const threads = yield* ThreadManagementService.ThreadManagementService;
118
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
119
const settings = yield* ServerSettings.ServerSettingsService;
120
+ const startRun = (
121
+ effect: EffectOutbox.OrchestrationEffectV2,
122
+ runId: RunId,
123
+ willRetry: boolean,
124
+ ) =>
125
+ providerTurnStart.start({ threadId: effect.threadId, runId, willRetry }).pipe(
126
+ Effect.catchCauseIf(
127
+ (cause) => !willRetry && !Cause.hasInterruptsOnly(cause),
128
+ (cause) =>
129
+ Effect.logWarning(
130
+ "Last run start attempt failed",
131
+ { effectId: effect.id, effectType: effect.request.type, runId },
132
+ cause,
133
+ ).pipe(
134
+ Effect.andThen(
135
+ Effect.suspend(() => {
136
+ const failure = Option.getOrUndefined(Cause.findErrorOption(cause));
137
+ // A failed read cannot identify the attempt. Keep retry ownership
138
+ // rather than settling whichever attempt happens to be current.
139
+ if (
140
+ !ProviderTurnStartService.isProviderTurnStartError(failure) ||
141
+ failure.attemptId === undefined ||
142
+ failure.rootNodeId === undefined
143
+ ) {
144
+ return Effect.failCause(cause);
145
+ }
146
+ return providerTurnStart.failStartingRun({
147
+ threadId: effect.threadId,
148
+ runId,
149
+ attemptId: failure.attemptId,
150
+ rootNodeId: failure.rootNodeId,
151
+ });
152
+ }),
153
+ ),
154
+ ),
155
+ ),
156
+ );
157
return OrchestrationEffectExecutorV2.of({
158
execute: Effect.fn("OrchestrationEffectExecutorV2.execute")(function (effect, options) {
159
const willRetry = options?.willRetry ?? false;
@@ -165,18 +217,16 @@ export const layerExecutor: Layer.Layer<
217
),
218
);
219
case "provider-turn.start":
168
- return providerTurnStart
169
- .start({ threadId: effect.threadId, runId: effect.request.runId, willRetry })
170
- .pipe(
171
- Effect.mapError(
172
- (cause) =>
173
- new OrchestrationEffectExecutionError({
174
- effectId: effect.id,
175
- effectType: effect.request.type,
176
- cause,
177
- }),
178
- ),
179
- );
220
+ return startRun(effect, effect.request.runId, willRetry).pipe(
221
+ Effect.mapError(
222
+ (cause) =>
223
+ new OrchestrationEffectExecutionError({
224
+ effectId: effect.id,
225
+ effectType: effect.request.type,
226
+ cause,
227
+ }),
228
+ ),
229
+ );
230
case "provider-turn.interrupt": {
231
const request = effect.request;
232
return providerTurnControl
@@ -414,11 +464,8 @@ export const layerExecutor: Layer.Layer<
464
: Effect.void,
465
),
466
Effect.andThen(
417
- providerTurnStart.start({
418
- threadId: effect.threadId,
419
- runId: effect.request.runId,
420
- willRetry,
421
- }),
467
+ // Failed interruption or detach must not report the old process stopped.
468
+ startRun(effect, effect.request.runId, willRetry),
469
),
470
Effect.mapError(
471
(cause) =>
@@ -906,7 +953,8 @@ export const layerWithOptions = (
953
? yield* outbox
954
.succeed({ effectId: effect.id, workerId })
955
.pipe(Effect.onError((cause) => terminalizeClaim(effect, cause)))
909
- : effect.attemptCount >= maxAttempts
956
+ : effect.attemptCount >= maxAttempts &&
957
+ (!ownsStartingRun(effect.request.type) || cannotSettleRun(exit.cause))
958
? yield* outbox
959
.fail({ effectId: effect.id, workerId, error })
960
.pipe(Effect.onError((cause) => terminalizeClaim(effect, cause)))
apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts
+513
-29
@@ -4,6 +4,7 @@ import { expect, it, vi } from "vite-plus/test";
4
import { it as effectIt } from "@effect/vitest";
5
import {
6
CheckpointScopeId,
7
+ CommandId,
8
ContextHandoffId,
9
ContextTransferId,
10
GitCommandError,
@@ -33,8 +34,14 @@ import * as Option from "effect/Option";
34
import * as Path from "effect/Path";
35
import * as PlatformError from "effect/PlatformError";
36
import * as Schema from "effect/Schema";
37
+import * as Stream from "effect/Stream";
38
+import * as TestClock from "effect/testing/TestClock";
39
40
import * as ServerConfig from "../config.ts";
41
+import * as McpAppModelContext from "../mcpApps/McpAppModelContext.ts";
42
+import * as PreviewAgentActivity from "../preview/PreviewAgentActivity.ts";
43
+import * as CheckpointService from "./CheckpointService.ts";
44
+import * as ProviderEventIngestor from "./ProviderEventIngestor.ts";
45
import {
46
withCheckpointRestoreReservation,
47
withWorkspaceLease,
@@ -52,6 +59,16 @@ import * as ServerSettings from "../serverSettings.ts";
59
import * as ContextHandoffService from "./ContextHandoffService.ts";
60
import { withInlineContext } from "./ContextHandoffDelivery.ts";
61
import * as EventSink from "./EventSink.ts";
62
+import * as SqlitePersistence from "../persistence/Sqlite.ts";
63
+import * as EffectOutbox from "./EffectOutbox.ts";
64
+import * as EffectWorker from "./EffectWorker.ts";
65
+import * as CheckpointRollbackService from "./CheckpointRollbackService.ts";
66
+import * as ProviderTurnControlService from "./ProviderTurnControlService.ts";
67
+import * as RunFinalizationService from "./RunFinalizationService.ts";
68
+import * as RuntimeRequestService from "./RuntimeRequestService.ts";
69
+import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
70
+import * as ThreadManagementService from "./ThreadManagementService.ts";
71
+import * as ThreadTitleRegenerationService from "./ThreadTitleRegenerationService.ts";
72
import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
73
import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
74
import * as ProjectionStore from "./ProjectionStore.ts";
@@ -394,6 +411,7 @@ function makeLocalCommandHarness(input: {
411
readonly interruptRunBeforeOpenFailure?: boolean;
412
readonly writeFailure?: unknown;
413
readonly failReadsAfterRunning?: boolean;
414
+ readonly missingProviderSession?: boolean;
415
readonly recoveryReadFailure?: string;
416
readonly previousTurn?: {
417
readonly runStatus: "failed" | "completed" | "interrupted";
@@ -458,7 +476,7 @@ function makeLocalCommandHarness(input: {
476
input.nativeGoals === undefined ? "codex" : (input.goalDriver ?? "opencode"),
477
),
478
providerInstanceId: newInstanceId,
461
- providerSessionId,
479
+ providerSessionId: input.missingProviderSession === true ? null : providerSessionId,
480
appThreadId: threadId,
481
ownerNodeId: null,
482
nativeThreadRef: null,
@@ -1098,7 +1116,9 @@ function makeLocalCommandHarness(input: {
1116
: Effect.die("A local command must not open a native session."),
1117
);
1118
const startRootRun = vi.fn<
1101
- (input: RunExecutionService.RunExecutionServiceV2StartRootRunInput) => Effect.Effect<void>
1119
+ (
1120
+ input: RunExecutionService.RunExecutionServiceV2StartRootRunInput,
1121
+ ) => Effect.Effect<void, RunExecutionService.RunExecutionServiceV2Error>
1122
>((runInput) =>
1123
input.sideChat !== undefined
1124
? (runInput.resume
@@ -1133,29 +1153,30 @@ function makeLocalCommandHarness(input: {
1153
}),
1154
),
1155
);
1136
- const writeIfRunCurrent = vi.fn(({ events: incoming, activeAttemptId, expectedStatus }) =>
1137
- "writeFailure" in input
1138
- ? Effect.fail(
1139
- new EventSink.EventSinkWriteError({
1140
- eventCount: incoming.length,
1141
- cause: input.writeFailure,
1142
- }),
1143
- )
1144
- : Effect.sync(() => {
1145
- const current = projection.runs.find((candidate) => candidate.id === runId);
1146
- const committed =
1147
- current !== undefined &&
1148
- current.activeAttemptId === activeAttemptId &&
1149
- current.status === expectedStatus;
1150
- if (committed) {
1151
- for (const event of incoming) {
1152
- expect(isDomainEvent(event)).toBe(true);
1153
- events.push(event);
1154
- projection = ProjectionStore.applyToProjection(projection, event);
1156
+ const writeIfRunCurrent = vi.fn<EventSink.EventSinkV2["Service"]["writeIfRunCurrent"]>(
1157
+ ({ events: incoming, activeAttemptId, expectedStatus }) =>
1158
+ "writeFailure" in input
1159
+ ? Effect.fail(
1160
+ new EventSink.EventSinkWriteError({
1161
+ eventCount: incoming.length,
1162
+ cause: input.writeFailure,
1163
+ }),
1164
+ )
1165
+ : Effect.sync(() => {
1166
+ const current = projection.runs.find((candidate) => candidate.id === runId);
1167
+ const committed =
1168
+ current !== undefined &&
1169
+ current.activeAttemptId === activeAttemptId &&
1170
+ current.status === expectedStatus;
1171
+ if (committed) {
1172
+ for (const event of incoming) {
1173
+ expect(isDomainEvent(event)).toBe(true);
1174
+ events.push(event);
1175
+ projection = ProjectionStore.applyToProjection(projection, event);
1176
+ }
1177
}
1156
- }
1157
- return { committed, storedEvents: [] };
1158
- }),
1178
+ return { committed, storedEvents: [] };
1179
+ }),
1180
);
1181
const resolveRuntimePolicy = vi.fn(
1182
({ thread }: Parameters<RuntimePolicy.RuntimePolicyV2Shape["resolve"]>[0]) =>
@@ -1199,6 +1220,12 @@ function makeLocalCommandHarness(input: {
1220
),
1221
}),
1222
);
1223
+ const getRun = vi.fn<ProjectionStore.ProjectionStoreV2Shape["getRun"]>((_threadId, id) =>
1224
+ Effect.as(
1225
+ failReadIfRunning,
1226
+ projection.runs.find((candidate) => candidate.id === id),
1227
+ ),
1228
+ );
1229
const layer = ProviderTurnStart.layer.pipe(
1230
Layer.provide(
1231
Layer.mergeAll(
@@ -1250,11 +1277,7 @@ function makeLocalCommandHarness(input: {
1277
getThreadRecords,
1278
getTurnStartContext,
1279
getRuntimeRecoveryProjection,
1253
- getRun: (_threadId, id) =>
1254
- Effect.as(
1255
- failReadIfRunning,
1256
- projection.runs.find((candidate) => candidate.id === id),
1257
- ),
1280
+ getRun,
1281
getTurnStartHistory: () =>
1282
input.sideChat !== undefined
1283
? Effect.succeed([])
@@ -1295,10 +1318,13 @@ function makeLocalCommandHarness(input: {
1318
),
1319
);
1320
return {
1321
+ layer,
1322
+ interruptRun,
1323
settledRuns,
1324
getThreadRecords,
1325
getTurnStartContext,
1326
getRuntimeRecoveryProjection,
1327
+ getRun,
1328
setProjection: (next: OrchestrationV2ThreadProjection) => {
1329
projection = next;
1330
},
@@ -1330,6 +1356,464 @@ function makeLocalCommandHarness(input: {
1356
};
1357
}
1358
1359
+function layerStartEffectWorker(
1360
+ harness: ReturnType<typeof makeLocalCommandHarness>,
1361
+ control: Partial<ProviderTurnControlService.ProviderTurnControlServiceV2["Service"]> = {},
1362
+) {
1363
+ const executor = EffectWorker.layerExecutor.pipe(
1364
+ Layer.provide(
1365
+ Layer.mergeAll(
1366
+ harness.layer,
1367
+ Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({}),
1368
+ Layer.mock(ProviderTurnControlService.ProviderTurnControlServiceV2)(control),
1369
+ Layer.mock(RunFinalizationService.RunFinalizationService)({}),
1370
+ Layer.mock(CheckpointRollbackService.CheckpointRollbackServiceV2)({}),
1371
+ Layer.mock(RuntimeRequestService.RuntimeRequestServiceV2)({}),
1372
+ Layer.mock(ThreadTitleRegenerationService.ThreadTitleRegenerationService)({}),
1373
+ Layer.mock(ThreadManagementService.ThreadManagementService)({}),
1374
+ Layer.mock(ProjectionStore.ProjectionStoreV2)({}),
1375
+ ServerSettings.layerTest(),
1376
+ ),
1377
+ ),
1378
+ );
1379
+ return EffectWorker.layerWithOptions({ maxAttempts: 2 }).pipe(
1380
+ Layer.provideMerge(
1381
+ Layer.merge(executor, EffectOutbox.layer.pipe(Layer.provide(SqlitePersistence.layerMemory))),
1382
+ ),
1383
+ );
1384
+}
1385
+
1386
+const enqueueStart = (
1387
+ projection: OrchestrationV2ThreadProjection,
1388
+ request?: EffectOutbox.OrchestrationEffectRequestV2,
1389
+) =>
1390
+ Effect.gen(function* () {
1391
+ const outbox = yield* EffectOutbox.EffectOutboxV2;
1392
+ const run = projection.runs.at(-1)!;
1393
+ yield* outbox.enqueue([
1394
+ {
1395
+ id: "effect:failed-start",
1396
+ commandId: CommandId.make("command:failed-start"),
1397
+ threadId: run.threadId,
1398
+ request: request ?? { type: "provider-turn.start", runId: run.id },
1399
+ },
1400
+ ]);
1401
+ return { storedEffect: outbox.get("effect:failed-start").pipe(Effect.map(Option.getOrThrow)) };
1402
+ });
1403
+
1404
+effectIt.effect(
1405
+ "settles an exhausted start effect without claiming an external turn stopped",
1406
+ () => {
1407
+ const harness = makeLocalCommandHarness({ text: "Continue", missingProviderSession: true });
1408
+ const projection = harness.projection();
1409
+ const previousAttempt = {
1410
+ ...projection.attempts[0]!,
1411
+ id: RunAttemptId.make("superseded-attempt"),
1412
+ rootNodeId: NodeId.make("superseded-root"),
1413
+ status: "superseded" as const,
1414
+ };
1415
+ const externalTurn = {
1416
+ id: ProviderTurnId.make("still-running-provider-turn"),
1417
+ providerThreadId: projection.providerThreads[0]!.id,
1418
+ nodeId: previousAttempt.rootNodeId,
1419
+ runAttemptId: previousAttempt.id,
1420
+ nativeTurnRef: null,
1421
+ ordinal: 1,
1422
+ status: "running" as const,
1423
+ startedAt: projection.updatedAt,
1424
+ completedAt: null,
1425
+ };
1426
+ harness.setProjection({
1427
+ ...projection,
1428
+ attempts: [
1429
+ previousAttempt,
1430
+ { ...projection.attempts[0]!, attemptOrdinal: 2, reason: "steering_restart" },
1431
+ ],
1432
+ providerTurns: [externalTurn],
1433
+ });
1434
+
1435
+ return Effect.gen(function* () {
1436
+ const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
1437
+ const { storedEffect } = yield* enqueueStart(harness.projection());
1438
+ expect(yield* worker.drain()).toBe(1);
1439
+ expect(yield* storedEffect).toMatchObject({ status: "pending", attemptCount: 1 });
1440
+ expect(harness.projection().runs.at(-1)?.status).toBe("starting");
1441
+ expect(harness.events).toEqual([]);
1442
+
1443
+ yield* TestClock.adjust("100 millis");
1444
+ expect(yield* worker.drain()).toBe(1);
1445
+ expect(yield* storedEffect).toMatchObject({ status: "succeeded", attemptCount: 2 });
1446
+ expect(harness.projection().runs.at(-1)).toMatchObject({ status: "failed", startedAt: null });
1447
+ expect(harness.projection().attempts.at(-1)).toMatchObject({
1448
+ status: "failed",
1449
+ startedAt: null,
1450
+ });
1451
+ expect(harness.projection().nodes[0]).toMatchObject({ status: "failed", startedAt: null });
1452
+ expect(harness.projection().turnItems).toMatchObject([
1453
+ {
1454
+ type: "error",
1455
+ status: "failed",
1456
+ title: "Turn failed to start",
1457
+ failure: {
1458
+ class: "unknown",
1459
+ message:
1460
+ "T3 Code could not start this turn. Send the message again; if it keeps failing, check the server logs.",
1461
+ },
1462
+ },
1463
+ ]);
1464
+ expect(harness.open).not.toHaveBeenCalled();
1465
+ expect(harness.startRootRun).not.toHaveBeenCalled();
1466
+ expect(harness.projection().providerTurns).toEqual([externalTurn]);
1467
+ }).pipe(Effect.provide(layerStartEffectWorker(harness)));
1468
+ },
1469
+);
1470
+
1471
+effectIt.effect("retries a failed settlement past the start budget until it is persisted", () => {
1472
+ const harness = makeLocalCommandHarness({ text: "Continue", missingProviderSession: true });
1473
+ harness.writeIfRunCurrent.mockImplementationOnce(({ events }) =>
1474
+ Effect.fail(
1475
+ new EventSink.EventSinkWriteError({
1476
+ eventCount: events.length,
1477
+ cause: "storage unavailable",
1478
+ }),
1479
+ ),
1480
+ );
1481
+
1482
+ return Effect.gen(function* () {
1483
+ const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
1484
+ const { storedEffect } = yield* enqueueStart(harness.projection());
1485
+ yield* worker.drain();
1486
+ yield* TestClock.adjust("100 millis");
1487
+ expect(yield* worker.drain()).toBe(1);
1488
+ expect(yield* storedEffect).toMatchObject({ status: "pending", attemptCount: 2 });
1489
+ expect(harness.projection().runs.at(-1)?.status).toBe("starting");
1490
+ expect(harness.events).toEqual([]);
1491
+
1492
+ yield* TestClock.adjust("200 millis");
1493
+ expect(yield* worker.drain()).toBe(1);
1494
+ expect(yield* storedEffect).toMatchObject({ status: "succeeded", attemptCount: 3 });
1495
+ expect(harness.projection().runs.at(-1)?.status).toBe("failed");
1496
+ expect(harness.projection().turnItems).toHaveLength(1);
1497
+ }).pipe(Effect.provide(layerStartEffectWorker(harness)));
1498
+});
1499
+
1500
+effectIt.effect.each(["interrupt", "start"] as const)(
1501
+ "keeps a failed restart %s separate from starting-run settlement",
1502
+ (stage) => {
1503
+ const harness = makeLocalCommandHarness({ text: "Continue", missingProviderSession: true });
1504
+ const projection = harness.projection();
1505
+ const run = projection.runs.at(-1)!;
1506
+ const providerTurnId = ProviderTurnId.make("restart-interrupted-turn");
1507
+ return Effect.gen(function* () {
1508
+ const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
1509
+ const { storedEffect } = yield* enqueueStart(projection, {
1510
+ type: "provider-turn.restart",
1511
+ runId: run.id,
1512
+ providerSessionId: ProviderSessionId.make("restart-interrupted-session"),
1513
+ providerThreadId: projection.providerThreads[0]!.id,
1514
+ providerTurnId,
1515
+ interruptedAttemptId: RunAttemptId.make("restart-interrupted-attempt"),
1516
+ });
1517
+ yield* worker.drain();
1518
+ yield* TestClock.adjust("100 millis");
1519
+ expect(yield* worker.drain()).toBe(1);
1520
+ if (stage === "interrupt") {
1521
+ yield* TestClock.adjust("200 millis");
1522
+ expect(yield* worker.drain()).toBe(1);
1523
+ expect(yield* storedEffect).toMatchObject({ status: "pending", attemptCount: 3 });
1524
+ expect(harness.projection().runs.at(-1)?.status).toBe("starting");
1525
+ expect(harness.events).toEqual([]);
1526
+ expect(harness.writeIfRunCurrent).not.toHaveBeenCalled();
1527
+ } else {
1528
+ expect(yield* storedEffect).toMatchObject({ status: "succeeded", attemptCount: 2 });
1529
+ expect(harness.projection().runs.at(-1)?.status).toBe("failed");
1530
+ expect(harness.projection().turnItems).toMatchObject([
1531
+ { type: "error", title: "Turn failed to start" },
1532
+ ]);
1533
+ }
1534
+ }).pipe(
1535
+ Effect.provide(
1536
+ layerStartEffectWorker(harness, {
1537
+ interruptAndAwaitTerminal: () =>
1538
+ stage === "interrupt"
1539
+ ? Effect.fail(
1540
+ new ProviderTurnControlService.ProviderTurnControlError({
1541
+ threadId: run.threadId,
1542
+ operation: "restart",
1543
+ providerTurnId,
1544
+ cause: "provider interrupt failed",
1545
+ }),
1546
+ )
1547
+ : Effect.void,
1548
+ }),
1549
+ ),
1550
+ );
1551
+ },
1552
+);
1553
+
1554
+effectIt.effect.each(["Stop", "a replacement attempt"] as const)(
1555
+ "does not overwrite %s committed during the final start failure write",
1556
+ (change) => {
1557
+ const harness = makeLocalCommandHarness({ text: "Continue", missingProviderSession: true });
1558
+ const write = harness.writeIfRunCurrent.getMockImplementation()!;
1559
+ let protectedProjection = harness.projection();
1560
+ harness.writeIfRunCurrent.mockImplementationOnce((input) =>
1561
+ Effect.suspend(() => {
1562
+ const projection = harness.projection();
1563
+ if (change === "Stop") harness.interruptRun();
1564
+ else {
1565
+ harness.setProjection({
1566
+ ...projection,
1567
+ runs: projection.runs.map((run) => ({
1568
+ ...run,
1569
+ activeAttemptId: RunAttemptId.make("replacement-attempt"),
1570
+ rootNodeId: NodeId.make("replacement-root"),
1571
+ })),
1572
+ });
1573
+ }
1574
+ protectedProjection = harness.projection();
1575
+ return write(input);
1576
+ }),
1577
+ );
1578
+
1579
+ return Effect.gen(function* () {
1580
+ const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
1581
+ const { storedEffect } = yield* enqueueStart(harness.projection());
1582
+ yield* worker.drain();
1583
+ yield* TestClock.adjust("100 millis");
1584
+ expect(yield* worker.drain()).toBe(1);
1585
+ expect(harness.writeIfRunCurrent).toHaveBeenCalledOnce();
1586
+ expect(harness.projection()).toEqual(protectedProjection);
1587
+ expect(harness.events).toEqual([]);
1588
+ expect(yield* storedEffect).toMatchObject({ status: "succeeded" });
1589
+ }).pipe(Effect.provide(layerStartEffectWorker(harness)));
1590
+ },
1591
+);
1592
+
1593
+effectIt.effect("preserves a replacement committed after the final pre-running start read", () => {
1594
+ const harness = makeLocalCommandHarness({ text: "Continue", missingProviderSession: true });
1595
+ const read = harness.getTurnStartContext.getMockImplementation()!;
1596
+ let reads = 0;
1597
+ let replacement: OrchestrationV2ThreadProjection;
1598
+ harness.getTurnStartContext.mockImplementation(() =>
1599
+ Effect.suspend(() => {
1600
+ const snapshot = read();
1601
+ reads++;
1602
+ if (reads === 2) {
1603
+ const projection = harness.projection();
1604
+ const attemptId = RunAttemptId.make("replacement-before-fallback");
1605
+ const rootNodeId = NodeId.make("replacement-root-before-fallback");
1606
+ replacement = {
1607
+ ...projection,
1608
+ runs: projection.runs.map((run) => ({ ...run, activeAttemptId: attemptId, rootNodeId })),
1609
+ attempts: [
1610
+ ...projection.attempts,
1611
+ { ...projection.attempts[0]!, id: attemptId, rootNodeId },
1612
+ ],
1613
+ nodes: [...projection.nodes, { ...projection.nodes[0]!, id: rootNodeId }],
1614
+ };
1615
+ harness.setProjection(replacement);
1616
+ }
1617
+ return snapshot;
1618
+ }),
1619
+ );
1620
+ return Effect.gen(function* () {
1621
+ const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
1622
+ const { storedEffect } = yield* enqueueStart(harness.projection());
1623
+ yield* worker.drain();
1624
+ yield* TestClock.adjust("100 millis");
1625
+ expect(yield* worker.drain()).toBe(1);
1626
+ expect(harness.projection()).toEqual(replacement!);
1627
+ expect(harness.events).toEqual([]);
1628
+ expect(yield* storedEffect).toMatchObject({ status: "succeeded", attemptCount: 2 });
1629
+ }).pipe(Effect.provide(layerStartEffectWorker(harness)));
1630
+});
1631
+
1632
+effectIt.effect("preserves a replacement committed before exhausted start settlement reads", () =>
1633
+ Effect.gen(function* () {
1634
+ const harness = makeLocalCommandHarness({ text: "Continue", failReadsAfterRunning: true });
1635
+ harness.getRuntimeRecoveryProjection.mockImplementation(() =>
1636
+ Effect.succeed(harness.projection()),
1637
+ );
1638
+ harness.getRun.mockImplementation((_threadId, id) =>
1639
+ Effect.succeed(harness.projection().runs.find((candidate) => candidate.id === id)),
1640
+ );
1641
+ const projection = harness.projection();
1642
+ const run = projection.runs[0]!;
1643
+ const replacementAttemptId = RunAttemptId.make("replacement-before-settlement");
1644
+ const replacementRootId = NodeId.make("replacement-root-before-settlement");
1645
+ const providerTurn = {
1646
+ id: ProviderTurnId.make("old-start-provider-turn"),
1647
+ providerThreadId: run.providerThreadId!,
1648
+ nodeId: run.rootNodeId!,
1649
+ runAttemptId: run.activeAttemptId!,
1650
+ nativeTurnRef: null,
1651
+ ordinal: 1,
1652
+ status: "running" as const,
1653
+ startedAt: projection.updatedAt,
1654
+ completedAt: null,
1655
+ };
1656
+ const turnIngested = yield* Deferred.make<void>();
1657
+ const storageFailure = new EventSink.EventSinkWriteError({
1658
+ eventCount: 5,
1659
+ cause: "terminal-write ownership query unavailable",
1660
+ });
1661
+ const write = harness.writeIfRunCurrent.getMockImplementation()!;
1662
+ harness.writeIfRunCurrent.mockImplementation((input) =>
1663
+ input.expectedStatus === "running" ? Effect.fail(storageFailure) : write(input),
1664
+ );
1665
+ harness.open.mockImplementationOnce(() =>
1666
+ Effect.fail(
1667
+ new ProviderSessionManager.ProviderSessionOpenError({
1668
+ instanceId: harness.newInstanceId,
1669
+ providerSessionId: projection.providerThreads[0]!.providerSessionId!,
1670
+ cause: "transient session failure",
1671
+ }),
1672
+ ),
1673
+ );
1674
+ const executionLayer = RunExecutionService.layer.pipe(
1675
+ Layer.provide(
1676
+ Layer.mergeAll(
1677
+ McpAppModelContext.layerEmpty,
1678
+ PreviewAgentActivity.layer,
1679
+ Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }),
1680
+ Layer.mock(TurnItemPositionStore.TurnItemPositionStoreV2)({}),
1681
+ Layer.mock(EventSink.EventSinkV2)({
1682
+ consumeProviderEvents: (source, process, after) =>
1683
+ source.pipe(
1684
+ Stream.mapEffect((event) => process(event).pipe(Effect.andThen(after))),
1685
+ Stream.takeUntil(Boolean),
1686
+ Stream.runDrain,
1687
+ ),
1688
+ writeIfRunCurrent: harness.writeIfRunCurrent,
1689
+ }),
1690
+ IdAllocator.layer,
1691
+ ThreadCommandExecutor.layer,
1692
+ Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
1693
+ ingestNormalized: () =>
1694
+ Effect.sync(() => {
1695
+ harness.setProjection({ ...harness.projection(), providerTurns: [providerTurn] });
1696
+ }).pipe(Effect.andThen(Deferred.succeed(turnIngested, undefined)), Effect.as([])),
1697
+ }),
1698
+ ServerSettings.layerTest(),
1699
+ ),
1700
+ ),
1701
+ );
1702
+ let protectedProjection = projection;
1703
+ let escapedStartError: RunExecutionService.RunExecutionServiceV2Error | undefined;
1704
+ return yield* Effect.gen(function* () {
1705
+ const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
1706
+ const outbox = yield* EffectOutbox.EffectOutboxV2;
1707
+ harness.startRootRun.mockImplementation((input) =>
1708
+ Effect.gen(function* () {
1709
+ const execution = yield* RunExecutionService.RunExecutionServiceV2;
1710
+ yield* execution.startRootRun({
1711
+ ...input,
1712
+ session: {
1713
+ ...input.session,
1714
+ events: Stream.concat(
1715
+ Stream.succeed({
1716
+ type: "provider_turn.updated" as const,
1717
+ driver: input.providerThread.driver,
1718
+ threadId: run.threadId,
1719
+ providerTurn,
1720
+ }),
1721
+ Stream.never,
1722
+ ),
1723
+ startTurn: () =>
1724
+ Effect.gen(function* () {
1725
+ yield* Deferred.await(turnIngested);
1726
+ const current = harness.projection();
1727
+ expect(current.runs[0]?.status).toBe("running");
1728
+ // Steering commits a new attempt while the original start still owns the lane.
1729
+ harness.setProjection({
1730
+ ...current,
1731
+ runs: [
1732
+ {
1733
+ ...current.runs[0]!,
1734
+ status: "starting",
1735
+ activeAttemptId: replacementAttemptId,
1736
+ rootNodeId: replacementRootId,
1737
+ },
1738
+ ],
1739
+ attempts: [
1740
+ {
1741
+ ...current.attempts[0]!,
1742
+ status: "superseded",
1743
+ completedAt: current.updatedAt,
1744
+ },
1745
+ {
1746
+ ...projection.attempts[0]!,
1747
+ id: replacementAttemptId,
1748
+ rootNodeId: replacementRootId,
1749
+ attemptOrdinal: 2,
1750
+ reason: "steering_restart",
1751
+ },
1752
+ ],
1753
+ nodes: [
1754
+ {
1755
+ ...current.nodes[0]!,
1756
+ status: "interrupted",
1757
+ completedAt: current.updatedAt,
1758
+ },
1759
+ {
1760
+ ...projection.nodes[0]!,
1761
+ id: replacementRootId,
1762
+ rootNodeId: replacementRootId,
1763
+ },
1764
+ ],
1765
+ });
1766
+ protectedProjection = harness.projection();
1767
+ yield* outbox
1768
+ .enqueue([
1769
+ {
1770
+ id: "effect:replacement-restart",
1771
+ commandId: CommandId.make("command:replacement-restart"),
1772
+ threadId: run.threadId,
1773
+ request: {
1774
+ type: "provider-turn.restart",
1775
+ runId: run.id,
1776
+ providerSessionId: input.providerSessionId,
1777
+ providerThreadId: run.providerThreadId!,
1778
+ providerTurnId: providerTurn.id,
1779
+ interruptedAttemptId: input.attemptId,
1780
+ },
1781
+ },
1782
+ ])
1783
+ .pipe(Effect.orDie);
1784
+ return yield* new ProviderAdapter.ProviderAdapterTurnStartError({
1785
+ driver: input.session.driver,
1786
+ threadId: run.threadId,
1787
+ providerThreadId: run.providerThreadId!,
1788
+ runId: run.id,
1789
+ cause: "provider start failed after publishing its turn",
1790
+ });
1791
+ }),
1792
+ },
1793
+ });
1794
+ }).pipe(
1795
+ Effect.tapError((error) =>
1796
+ Effect.sync(() => {
1797
+ escapedStartError = error;
1798
+ }),
1799
+ ),
1800
+ Effect.provide(executionLayer.pipe(Layer.orDie)),
1801
+ ),
1802
+ );
1803
+ const { storedEffect } = yield* enqueueStart(projection);
1804
+ expect(yield* worker.drain()).toBe(1);
1805
+ yield* TestClock.adjust("100 millis");
1806
+ expect(yield* worker.drain(1)).toBe(1);
1807
+ expect(escapedStartError).toMatchObject({ cause: { write: storageFailure } });
1808
+ expect(yield* outbox.get("effect:replacement-restart")).toMatchObject({
1809
+ value: { status: "pending", attemptCount: 0 },
1810
+ });
1811
+ expect(yield* storedEffect).toMatchObject({ status: "succeeded", attemptCount: 2 });
1812
+ expect(harness.projection()).toEqual(protectedProjection);
1813
+ }).pipe(Effect.provide(layerStartEffectWorker(harness)));
1814
+ }),
1815
+);
1816
+
1817
effectIt.effect(
1818
"recovers the local or pushed thread tip, else the current branch, remote default, or detached HEAD",
1819
() =>
apps/server/src/orchestration-v2/ProviderTurnStartService.ts
+241
-118
@@ -13,8 +13,11 @@ import {
13
isCompactionCommandMessage,
14
parseGoalCommand,
15
RunId,
16
+ RunAttemptId,
17
+ NodeId,
18
ThreadId,
19
} from "@t3tools/contracts";
20
+import * as Cause from "effect/Cause";
21
import * as Context from "effect/Context";
22
import * as DateTime from "effect/DateTime";
23
import * as Effect from "effect/Effect";
@@ -75,6 +78,8 @@ export class ProviderTurnStartError extends Schema.TaggedError<ProviderTurnStart
78
{
79
runId: RunId,
80
cause: Schema.optional(Schema.Defect()),
81
+ attemptId: Schema.optional(RunAttemptId),
82
+ rootNodeId: Schema.optional(NodeId),
83
},
84
) {}
85
@@ -86,7 +91,7 @@ const NON_COMPLETING_TERMINAL_RUN_STATUSES: ReadonlySet<OrchestrationV2Run["stat
91
"rolled_back",
92
]);
93
89
-const isProviderTurnStartError = Schema.is(ProviderTurnStartError);
94
+export const isProviderTurnStartError = Schema.is(ProviderTurnStartError);
95
const isProviderAdapterProtocolError = Schema.is(ProviderAdapter.ProviderAdapterProtocolError);
96
const isProviderAdapterOpenSessionError = Schema.is(
97
ProviderAdapter.ProviderAdapterOpenSessionError,
@@ -113,6 +118,21 @@ const refusedBeforePrompt = (error: unknown): boolean =>
118
error.detail === CLAUDE_QUERY_ENDED_BEFORE_TURN_DETAIL) ||
119
(Predicate.hasProperty(error, "cause") && refusedBeforePrompt(error.cause));
120
121
+/** Missing execution records cannot be repaired by retrying a start. */
122
+export class ProviderTurnStartRunStateMissingError extends Schema.TaggedError<ProviderTurnStartRunStateMissingError>()(
123
+ "ProviderTurnStartRunStateMissingError",
124
+ {
125
+ runId: RunId,
126
+ },
127
+) {}
128
+
129
+export const isProviderTurnStartRunStateMissingError = Schema.is(
130
+ ProviderTurnStartRunStateMissingError,
131
+);
132
+
133
+const START_GAVE_UP_MESSAGE =
134
+ "T3 Code could not start this turn. Send the message again; if it keeps failing, check the server logs.";
135
+
136
const startFailureMessage = (error: Error): string => {
137
const nestedCause = "cause" in error ? error.cause : undefined;
138
return (
@@ -131,6 +151,12 @@ export interface ProviderTurnStartServiceV2Shape {
151
readonly runId: RunId;
152
readonly willRetry?: boolean;
153
}) => Effect.Effect<void, ProviderTurnStartError>;
154
+ readonly failStartingRun: (input: {
155
+ readonly threadId: ThreadId;
156
+ readonly runId: RunId;
157
+ readonly attemptId: RunAttemptId;
158
+ readonly rootNodeId: NodeId;
159
+ }) => Effect.Effect<void, ProviderTurnStartError | ProviderTurnStartRunStateMissingError>;
160
}
161
162
export class ProviderTurnStartServiceV2 extends Context.Service<
@@ -289,11 +315,109 @@ export const layer: Layer.Layer<
315
};
316
};
317
292
- const start = Effect.fn("orchestrationV2.providerTurnStart.start")(function* (input: {
293
- readonly threadId: ThreadId;
294
- readonly runId: RunId;
295
- readonly willRetry?: boolean;
296
- }) {
318
+ const settleRunBeforeStart = Effect.fn("orchestrationV2.providerTurnStart.settleBeforeStart")(
319
+ function* (input: {
320
+ readonly run: OrchestrationV2Run;
321
+ readonly rootNode: OrchestrationV2ExecutionNode;
322
+ readonly attempt: OrchestrationV2RunAttempt;
323
+ readonly turnItems: ReadonlyArray<OrchestrationV2TurnItem>;
324
+ readonly signal: string;
325
+ readonly status: "completed" | "failed";
326
+ readonly now: DateTime.Utc;
327
+ readonly startedAt?: DateTime.Utc;
328
+ readonly providerInstanceId: OrchestrationV2Run["providerInstanceId"];
329
+ readonly itemProviderThreadId: OrchestrationV2ProviderThread["id"] | null;
330
+ readonly item:
331
+ | Pick<Extract<OrchestrationV2TurnItem, { type: "error" }>, "type" | "title" | "failure">
332
+ | Pick<
333
+ Extract<OrchestrationV2TurnItem, { type: "command_execution" }>,
334
+ "type" | "title" | "input" | "output" | "exitCode"
335
+ >;
336
+ readonly providerThreadUpdate?: OrchestrationV2ProviderThread;
337
+ }) {
338
+ const { run, rootNode, attempt, turnItems, now, status } = input;
339
+ const runId = run.id;
340
+ const started = input.startedAt === undefined ? {} : { startedAt: input.startedAt };
341
+ const item: OrchestrationV2TurnItem = {
342
+ id: idAllocator.derive.runSignalTurnItem({ runId, signal: input.signal }),
343
+ threadId: run.threadId,
344
+ runId,
345
+ nodeId: rootNode.id,
346
+ providerThreadId: input.itemProviderThreadId,
347
+ providerTurnId: null,
348
+ nativeItemRef: null,
349
+ parentItemId: null,
350
+ ordinal:
351
+ Math.max(
352
+ 0,
353
+ ...turnItems.filter((item) => item.runId === runId).map((item) => item.ordinal),
354
+ ) + 1,
355
+ status,
356
+ startedAt: now,
357
+ completedAt: now,
358
+ updatedAt: now,
359
+ ...input.item,
360
+ };
361
+ const eventPayloads = [
362
+ { type: "turn-item.updated", payload: item },
363
+ { type: "run.updated", payload: { ...run, status, ...started, completedAt: now } },
364
+ {
365
+ type: "run-attempt.updated",
366
+ payload: { ...attempt, status, ...started, completedAt: now },
367
+ },
368
+ {
369
+ type: "node.updated",
370
+ payload: { ...rootNode, status, ...started, completedAt: now },
371
+ },
372
+ ...(input.providerThreadUpdate === undefined
373
+ ? []
374
+ : [
375
+ {
376
+ type: "provider-thread.updated" as const,
377
+ payload: input.providerThreadUpdate,
378
+ },
379
+ ]),
380
+ ] as const;
381
+ const events = yield* Effect.forEach(eventPayloads, (event) =>
382
+ Effect.gen(function* () {
383
+ return {
384
+ ...event,
385
+ id: yield* idAllocator.allocate.event({ threadId: run.threadId }),
386
+ threadId: run.threadId,
387
+ runId,
388
+ nodeId: rootNode.id,
389
+ providerInstanceId: input.providerInstanceId,
390
+ occurredAt: now,
391
+ } satisfies OrchestrationV2DomainEvent;
392
+ }),
393
+ );
394
+ const written = yield* eventSink.writeIfRunCurrent({
395
+ threadId: run.threadId,
396
+ runId,
397
+ activeAttemptId: attempt.id,
398
+ expectedStatus: "starting",
399
+ events,
400
+ });
401
+ // A steering restart that cannot start still ends a run whose earlier turns counted.
402
+ if (written.committed) {
403
+ yield* turnAnalytics.runSettled({
404
+ runId,
405
+ status,
406
+ completedAt: now,
407
+ childrenRunning: false,
408
+ });
409
+ }
410
+ },
411
+ );
412
+
413
+ const start = Effect.fn("orchestrationV2.providerTurnStart.start")(function* (
414
+ input: {
415
+ readonly threadId: ThreadId;
416
+ readonly runId: RunId;
417
+ readonly willRetry?: boolean;
418
+ },
419
+ captureStartingRun: (run: OrchestrationV2Run) => void,
420
+ ) {
421
const { runId } = input;
422
const projection = yield* projectionStore.getTurnStartContext(input.threadId, runId);
423
const run = projection.runs.find((candidate) => candidate.id === runId);
@@ -303,6 +427,7 @@ export const layer: Layer.Layer<
427
if (run.status !== "starting") {
428
return;
429
}
430
+ captureStartingRun(run);
431
const rootNode = projection.nodes.find((candidate) => candidate.id === run.rootNodeId);
432
const attempt = projection.attempts.find((candidate) => candidate.id === run.activeAttemptId);
433
const providerThread = projection.providerThreads.find(
@@ -385,100 +510,6 @@ export const layer: Layer.Layer<
510
? projection.messages.find((source) => source.id === resumeSource.userMessageId)
511
: undefined;
512
const providerMessage = unsentResumeMessage ?? message;
388
- const settleRunBeforeStart = Effect.fn("orchestrationV2.providerTurnStart.settleBeforeStart")(
389
- function* (input: {
390
- readonly signal: string;
391
- readonly status: "completed" | "failed";
392
- readonly now: DateTime.Utc;
393
- readonly startedAt?: DateTime.Utc;
394
- readonly providerInstanceId: OrchestrationV2Run["providerInstanceId"];
395
- readonly itemProviderThreadId: OrchestrationV2ProviderThread["id"];
396
- readonly item:
397
- | Pick<
398
- Extract<OrchestrationV2TurnItem, { type: "error" }>,
399
- "type" | "title" | "failure"
400
- >
401
- | Pick<
402
- Extract<OrchestrationV2TurnItem, { type: "command_execution" }>,
403
- "type" | "title" | "input" | "output" | "exitCode"
404
- >;
405
- readonly providerThreadUpdate?: OrchestrationV2ProviderThread;
406
- }) {
407
- const { now, status } = input;
408
- const started = input.startedAt === undefined ? {} : { startedAt: input.startedAt };
409
- const item: OrchestrationV2TurnItem = {
410
- id: idAllocator.derive.runSignalTurnItem({ runId, signal: input.signal }),
411
- threadId: projection.thread.id,
412
- runId,
413
- nodeId: rootNode.id,
414
- providerThreadId: input.itemProviderThreadId,
415
- providerTurnId: null,
416
- nativeItemRef: null,
417
- parentItemId: null,
418
- ordinal:
419
- Math.max(
420
- 0,
421
- ...projection.turnItems
422
- .filter((item) => item.runId === runId)
423
- .map((item) => item.ordinal),
424
- ) + 1,
425
- status,
426
- startedAt: now,
427
- completedAt: now,
428
- updatedAt: now,
429
- ...input.item,
430
- };
431
- const eventPayloads = [
432
- { type: "turn-item.updated", payload: item },
433
- { type: "run.updated", payload: { ...run, status, ...started, completedAt: now } },
434
- {
435
- type: "run-attempt.updated",
436
- payload: { ...attempt, status, ...started, completedAt: now },
437
- },
438
- {
439
- type: "node.updated",
440
- payload: { ...rootNode, status, ...started, completedAt: now },
441
- },
442
- ...(input.providerThreadUpdate === undefined
443
- ? []
444
- : [
445
- {
446
- type: "provider-thread.updated" as const,
447
- payload: input.providerThreadUpdate,
448
- },
449
- ]),
450
- ] as const;
451
- const events = yield* Effect.forEach(eventPayloads, (event) =>
452
- Effect.gen(function* () {
453
- return {
454
- ...event,
455
- id: yield* idAllocator.allocate.event({ threadId: projection.thread.id }),
456
- threadId: projection.thread.id,
457
- runId,
458
- nodeId: rootNode.id,
459
- providerInstanceId: input.providerInstanceId,
460
- occurredAt: now,
461
- } satisfies OrchestrationV2DomainEvent;
462
- }),
463
- );
464
- const written = yield* eventSink.writeIfRunCurrent({
465
- threadId: projection.thread.id,
466
- runId,
467
- activeAttemptId: attempt.id,
468
- expectedStatus: "starting",
469
- events,
470
- });
471
- // A steering restart that cannot start still ends a run whose earlier turns counted.
472
- if (written.committed) {
473
- yield* turnAnalytics.runSettled({
474
- runId,
475
- status,
476
- completedAt: now,
477
- childrenRunning: false,
478
- });
479
- }
480
- },
481
- );
513
const settleStartFailure = Effect.fn("ProviderTurnStartService.settleStartFailure")(
514
function* (failed: {
515
readonly signal: string;
@@ -487,6 +518,10 @@ export const layer: Layer.Layer<
518
readonly message?: string;
519
}) {
520
yield* settleRunBeforeStart({
521
+ run,
522
+ rootNode,
523
+ attempt,
524
+ turnItems: projection.turnItems,
525
signal: failed.signal,
526
status: "failed",
527
now: yield* DateTime.now,
@@ -561,6 +596,10 @@ export const layer: Layer.Layer<
596
: undefined;
597
const status = failure === undefined ? "completed" : "failed";
598
yield* settleRunBeforeStart({
599
+ run,
600
+ rootNode,
601
+ attempt,
602
+ turnItems: projection.turnItems,
603
signal: isEmptyCompaction ? "empty-compaction" : "provider-sign-out",
604
status,
605
now,
@@ -1833,25 +1872,65 @@ export const layer: Layer.Layer<
1872
return ProviderTurnStartServiceV2.of({
1873
start: Effect.fn("ProviderTurnStartService.start")(
1874
function* (input) {
1836
- // Storage cleanup removes worktrees under the same lease. Only startup
1837
- // holds it; the provider turn runs after it is released.
1838
- const { thread } = yield* projectionStore.getThreadRecords(input.threadId, []);
1839
- const cwd =
1840
- thread.worktreePath ??
1841
- (yield* projects
1842
- .getById(thread.projectId, { includeDeleted: true })
1843
- .pipe(Effect.map((project) => Option.getOrUndefined(project)?.workspaceRoot)));
1844
- const providerTurn = yield* cwd === undefined
1845
- ? start(input)
1846
- : withTurnStartReservation(
1847
- fileSystem,
1848
- path,
1849
- path.resolve(cwd),
1850
- thread.worktreePath === null
1851
- ? start(input)
1852
- : withWorkspaceLease(path.resolve(thread.worktreePath), start(input)),
1875
+ let startingRun: OrchestrationV2Run | undefined;
1876
+ const captureStartingRun = (run: OrchestrationV2Run) => {
1877
+ startingRun = run;
1878
+ };
1879
+ return yield* Effect.gen(function* () {
1880
+ // Storage cleanup removes worktrees under the same lease. Only startup
1881
+ // holds it; the provider turn runs after it is released.
1882
+ const { thread } = yield* projectionStore.getThreadRecords(input.threadId, []);
1883
+ const cwd =
1884
+ thread.worktreePath ??
1885
+ (yield* projects
1886
+ .getById(thread.projectId, { includeDeleted: true })
1887
+ .pipe(Effect.map((project) => Option.getOrUndefined(project)?.workspaceRoot)));
1888
+ const providerTurn = yield* cwd === undefined
1889
+ ? start(input, captureStartingRun)
1890
+ : withTurnStartReservation(
1891
+ fileSystem,
1892
+ path,
1893
+ path.resolve(cwd),
1894
+ thread.worktreePath === null
1895
+ ? start(input, captureStartingRun)
1896
+ : withWorkspaceLease(
1897
+ path.resolve(thread.worktreePath),
1898
+ start(input, captureStartingRun),
1899
+ ),
1900
+ );
1901
+ if (providerTurn !== undefined) {
1902
+ // Run execution owns finalization after the running write. A later
1903
+ // failure must not settle a replacement attempt committed by steering.
1904
+ yield* providerTurn.pipe(
1905
+ Effect.catchCauseIf(
1906
+ (cause) => !Cause.hasInterruptsOnly(cause),
1907
+ (cause) =>
1908
+ Effect.logWarning(
1909
+ "Run execution failed after starting",
1910
+ { runId: input.runId },
1911
+ cause,
1912
+ ),
1913
+ ),
1914
);
1854
- if (providerTurn !== undefined) yield* providerTurn;
1915
+ }
1916
+ }).pipe(
1917
+ Effect.catchCauseIf(
1918
+ (cause) => !Cause.hasInterruptsOnly(cause),
1919
+ (cause) =>
1920
+ Effect.fail(
1921
+ new ProviderTurnStartError({
1922
+ runId: input.runId,
1923
+ cause: Cause.squash(cause),
1924
+ ...(startingRun?.activeAttemptId == null || startingRun.rootNodeId == null
1925
+ ? {}
1926
+ : {
1927
+ attemptId: startingRun.activeAttemptId,
1928
+ rootNodeId: startingRun.rootNodeId,
1929
+ }),
1930
+ }),
1931
+ ),
1932
+ ),
1933
+ );
1934
},
1935
(effect, input) =>
1936
effect.pipe(
@@ -1862,6 +1941,50 @@ export const layer: Layer.Layer<
1941
),
1942
),
1943
),
1944
+ failStartingRun: Effect.fn("ProviderTurnStartService.failStartingRun")(
1945
+ function* ({ threadId, runId, attemptId, rootNodeId }) {
1946
+ const projection = yield* projectionStore.getTurnStartContext(threadId, runId);
1947
+ const run = projection.runs.find((candidate) => candidate.id === runId);
1948
+ if (
1949
+ run === undefined ||
1950
+ run.status !== "starting" ||
1951
+ run.activeAttemptId !== attemptId ||
1952
+ run.rootNodeId !== rootNodeId
1953
+ )
1954
+ return;
1955
+ const rootNode = projection.nodes.find((candidate) => candidate.id === run.rootNodeId);
1956
+ const attempt = projection.attempts.find(
1957
+ (candidate) => candidate.id === run.activeAttemptId,
1958
+ );
1959
+ if (rootNode === undefined || attempt === undefined) {
1960
+ return yield* new ProviderTurnStartRunStateMissingError({ runId });
1961
+ }
1962
+ yield* settleRunBeforeStart({
1963
+ run,
1964
+ rootNode,
1965
+ attempt,
1966
+ turnItems: projection.turnItems,
1967
+ signal: "provider-turn-start-failure",
1968
+ status: "failed",
1969
+ now: yield* DateTime.now,
1970
+ providerInstanceId: run.providerInstanceId,
1971
+ itemProviderThreadId: run.providerThreadId,
1972
+ item: {
1973
+ type: "error",
1974
+ title: "Turn failed to start",
1975
+ failure: makeProviderFailure({ message: START_GAVE_UP_MESSAGE }),
1976
+ },
1977
+ });
1978
+ },
1979
+ (effect, { runId }) =>
1980
+ effect.pipe(
1981
+ Effect.mapError((cause) =>
1982
+ isProviderTurnStartRunStateMissingError(cause)
1983
+ ? cause
1984
+ : new ProviderTurnStartError({ runId, cause }),
1985
+ ),
1986
+ ),
1987
+ ),
1988
});
1989
}),
1990
);