@devries-cameron20 / t3code Mirror · read-only / commits / c76ba848ab
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 13:30 UTC

fix(server): a failed turn start no longer leaves the thread stuck starting (#14424)

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: SAPHID <saphid@users.noreply.github.com> Co-authored-by: Julius Marminge <julius0216@outlook.com>

Alex Southwell committed Nov 15, 2026 at 07:58 UTC c76ba848ab5d83e1babfd1db731bd5ce92ff4312
4 files changed +822 -166
apps/server/src/orchestration-v2/EffectWorker.test.ts
+1
@@ -126,6 +126,7 @@ function layerExecutorFor(input: {
126 Layer.succeed(
127 ProviderTurnStartService.ProviderTurnStartServiceV2,
128 ProviderTurnStartService.ProviderTurnStartServiceV2.of({
129 + failStartingRun: () => Effect.void,
130 start: () =>
131 Effect.gen(function* () {
132 yield* record("start");
apps/server/src/orchestration-v2/EffectWorker.ts
+67 -19
@@ -1,4 +1,4 @@
1 -import { CommandId } from "@t3tools/contracts";
1 +import { CommandId, type RunId } from "@t3tools/contracts";
2 import * as Cause from "effect/Cause";
3 import * as Context from "effect/Context";
4 import * as DateTime from "effect/DateTime";
@@ -44,6 +44,8 @@ export class OrchestrationEffectExecutionError extends Schema.TaggedError<Orches
44 },
45 ) {}
46
47 +const isOrchestrationEffectExecutionError = Schema.is(OrchestrationEffectExecutionError);
48 +
49 /**
50 * Do not apply this to `provider-turn.restart`: that compound effect also runs
51 * detach and start. Swallowing a start failure that happens to mention
@@ -62,6 +64,19 @@ function isNonRetryableProviderTurnControlFailure(effectType: string, errorText:
64 );
65 }
66
67 +// Keep an owner for starting runs when persistence or restart cleanup fails.
68 +const ownsStartingRun = (effectType: EffectOutbox.OrchestrationEffectV2["request"]["type"]) =>
69 + effectType === "provider-turn.start" || effectType === "provider-turn.restart";
70 +
71 +const cannotSettleRun = (cause: Cause.Cause<unknown>) =>
72 + Cause.findErrorOption(cause).pipe(
73 + Option.exists(
74 + (error) =>
75 + isOrchestrationEffectExecutionError(error) &&
76 + ProviderTurnStartService.isProviderTurnStartRunStateMissingError(error.cause),
77 + ),
78 + );
79 +
80 export interface OrchestrationEffectExecutorV2Shape {
81 readonly execute: (
82 effect: EffectOutbox.OrchestrationEffectV2,
@@ -102,6 +117,43 @@ export const layerExecutor: Layer.Layer<
117 const threads = yield* ThreadManagementService.ThreadManagementService;
118 const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
119 const settings = yield* ServerSettings.ServerSettingsService;
120 + const startRun = (
121 + effect: EffectOutbox.OrchestrationEffectV2,
122 + runId: RunId,
123 + willRetry: boolean,
124 + ) =>
125 + providerTurnStart.start({ threadId: effect.threadId, runId, willRetry }).pipe(
126 + Effect.catchCauseIf(
127 + (cause) => !willRetry && !Cause.hasInterruptsOnly(cause),
128 + (cause) =>
129 + Effect.logWarning(
130 + "Last run start attempt failed",
131 + { effectId: effect.id, effectType: effect.request.type, runId },
132 + cause,
133 + ).pipe(
134 + Effect.andThen(
135 + Effect.suspend(() => {
136 + const failure = Option.getOrUndefined(Cause.findErrorOption(cause));
137 + // A failed read cannot identify the attempt. Keep retry ownership
138 + // rather than settling whichever attempt happens to be current.
139 + if (
140 + !ProviderTurnStartService.isProviderTurnStartError(failure) ||
141 + failure.attemptId === undefined ||
142 + failure.rootNodeId === undefined
143 + ) {
144 + return Effect.failCause(cause);
145 + }
146 + return providerTurnStart.failStartingRun({
147 + threadId: effect.threadId,
148 + runId,
149 + attemptId: failure.attemptId,
150 + rootNodeId: failure.rootNodeId,
151 + });
152 + }),
153 + ),
154 + ),
155 + ),
156 + );
157 return OrchestrationEffectExecutorV2.of({
158 execute: Effect.fn("OrchestrationEffectExecutorV2.execute")(function (effect, options) {
159 const willRetry = options?.willRetry ?? false;
@@ -165,18 +217,16 @@ export const layerExecutor: Layer.Layer<
217 ),
218 );
219 case "provider-turn.start":
168 - return providerTurnStart
169 - .start({ threadId: effect.threadId, runId: effect.request.runId, willRetry })
170 - .pipe(
171 - Effect.mapError(
172 - (cause) =>
173 - new OrchestrationEffectExecutionError({
174 - effectId: effect.id,
175 - effectType: effect.request.type,
176 - cause,
177 - }),
178 - ),
179 - );
220 + return startRun(effect, effect.request.runId, willRetry).pipe(
221 + Effect.mapError(
222 + (cause) =>
223 + new OrchestrationEffectExecutionError({
224 + effectId: effect.id,
225 + effectType: effect.request.type,
226 + cause,
227 + }),
228 + ),
229 + );
230 case "provider-turn.interrupt": {
231 const request = effect.request;
232 return providerTurnControl
@@ -414,11 +464,8 @@ export const layerExecutor: Layer.Layer<
464 : Effect.void,
465 ),
466 Effect.andThen(
417 - providerTurnStart.start({
418 - threadId: effect.threadId,
419 - runId: effect.request.runId,
420 - willRetry,
421 - }),
467 + // Failed interruption or detach must not report the old process stopped.
468 + startRun(effect, effect.request.runId, willRetry),
469 ),
470 Effect.mapError(
471 (cause) =>
@@ -906,7 +953,8 @@ export const layerWithOptions = (
953 ? yield* outbox
954 .succeed({ effectId: effect.id, workerId })
955 .pipe(Effect.onError((cause) => terminalizeClaim(effect, cause)))
909 - : effect.attemptCount >= maxAttempts
956 + : effect.attemptCount >= maxAttempts &&
957 + (!ownsStartingRun(effect.request.type) || cannotSettleRun(exit.cause))
958 ? yield* outbox
959 .fail({ effectId: effect.id, workerId, error })
960 .pipe(Effect.onError((cause) => terminalizeClaim(effect, cause)))
apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts
+513 -29
@@ -4,6 +4,7 @@ import { expect, it, vi } from "vite-plus/test";
4 import { it as effectIt } from "@effect/vitest";
5 import {
6 CheckpointScopeId,
7 + CommandId,
8 ContextHandoffId,
9 ContextTransferId,
10 GitCommandError,
@@ -33,8 +34,14 @@ import * as Option from "effect/Option";
34 import * as Path from "effect/Path";
35 import * as PlatformError from "effect/PlatformError";
36 import * as Schema from "effect/Schema";
37 +import * as Stream from "effect/Stream";
38 +import * as TestClock from "effect/testing/TestClock";
39
40 import * as ServerConfig from "../config.ts";
41 +import * as McpAppModelContext from "../mcpApps/McpAppModelContext.ts";
42 +import * as PreviewAgentActivity from "../preview/PreviewAgentActivity.ts";
43 +import * as CheckpointService from "./CheckpointService.ts";
44 +import * as ProviderEventIngestor from "./ProviderEventIngestor.ts";
45 import {
46 withCheckpointRestoreReservation,
47 withWorkspaceLease,
@@ -52,6 +59,16 @@ import * as ServerSettings from "../serverSettings.ts";
59 import * as ContextHandoffService from "./ContextHandoffService.ts";
60 import { withInlineContext } from "./ContextHandoffDelivery.ts";
61 import * as EventSink from "./EventSink.ts";
62 +import * as SqlitePersistence from "../persistence/Sqlite.ts";
63 +import * as EffectOutbox from "./EffectOutbox.ts";
64 +import * as EffectWorker from "./EffectWorker.ts";
65 +import * as CheckpointRollbackService from "./CheckpointRollbackService.ts";
66 +import * as ProviderTurnControlService from "./ProviderTurnControlService.ts";
67 +import * as RunFinalizationService from "./RunFinalizationService.ts";
68 +import * as RuntimeRequestService from "./RuntimeRequestService.ts";
69 +import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
70 +import * as ThreadManagementService from "./ThreadManagementService.ts";
71 +import * as ThreadTitleRegenerationService from "./ThreadTitleRegenerationService.ts";
72 import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
73 import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
74 import * as ProjectionStore from "./ProjectionStore.ts";
@@ -394,6 +411,7 @@ function makeLocalCommandHarness(input: {
411 readonly interruptRunBeforeOpenFailure?: boolean;
412 readonly writeFailure?: unknown;
413 readonly failReadsAfterRunning?: boolean;
414 + readonly missingProviderSession?: boolean;
415 readonly recoveryReadFailure?: string;
416 readonly previousTurn?: {
417 readonly runStatus: "failed" | "completed" | "interrupted";
@@ -458,7 +476,7 @@ function makeLocalCommandHarness(input: {
476 input.nativeGoals === undefined ? "codex" : (input.goalDriver ?? "opencode"),
477 ),
478 providerInstanceId: newInstanceId,
461 - providerSessionId,
479 + providerSessionId: input.missingProviderSession === true ? null : providerSessionId,
480 appThreadId: threadId,
481 ownerNodeId: null,
482 nativeThreadRef: null,
@@ -1098,7 +1116,9 @@ function makeLocalCommandHarness(input: {
1116 : Effect.die("A local command must not open a native session."),
1117 );
1118 const startRootRun = vi.fn<
1101 - (input: RunExecutionService.RunExecutionServiceV2StartRootRunInput) => Effect.Effect<void>
1119 + (
1120 + input: RunExecutionService.RunExecutionServiceV2StartRootRunInput,
1121 + ) => Effect.Effect<void, RunExecutionService.RunExecutionServiceV2Error>
1122 >((runInput) =>
1123 input.sideChat !== undefined
1124 ? (runInput.resume
@@ -1133,29 +1153,30 @@ function makeLocalCommandHarness(input: {
1153 }),
1154 ),
1155 );
1136 - const writeIfRunCurrent = vi.fn(({ events: incoming, activeAttemptId, expectedStatus }) =>
1137 - "writeFailure" in input
1138 - ? Effect.fail(
1139 - new EventSink.EventSinkWriteError({
1140 - eventCount: incoming.length,
1141 - cause: input.writeFailure,
1142 - }),
1143 - )
1144 - : Effect.sync(() => {
1145 - const current = projection.runs.find((candidate) => candidate.id === runId);
1146 - const committed =
1147 - current !== undefined &&
1148 - current.activeAttemptId === activeAttemptId &&
1149 - current.status === expectedStatus;
1150 - if (committed) {
1151 - for (const event of incoming) {
1152 - expect(isDomainEvent(event)).toBe(true);
1153 - events.push(event);
1154 - projection = ProjectionStore.applyToProjection(projection, event);
1156 + const writeIfRunCurrent = vi.fn<EventSink.EventSinkV2["Service"]["writeIfRunCurrent"]>(
1157 + ({ events: incoming, activeAttemptId, expectedStatus }) =>
1158 + "writeFailure" in input
1159 + ? Effect.fail(
1160 + new EventSink.EventSinkWriteError({
1161 + eventCount: incoming.length,
1162 + cause: input.writeFailure,
1163 + }),
1164 + )
1165 + : Effect.sync(() => {
1166 + const current = projection.runs.find((candidate) => candidate.id === runId);
1167 + const committed =
1168 + current !== undefined &&
1169 + current.activeAttemptId === activeAttemptId &&
1170 + current.status === expectedStatus;
1171 + if (committed) {
1172 + for (const event of incoming) {
1173 + expect(isDomainEvent(event)).toBe(true);
1174 + events.push(event);
1175 + projection = ProjectionStore.applyToProjection(projection, event);
1176 + }
1177 }
1156 - }
1157 - return { committed, storedEvents: [] };
1158 - }),
1178 + return { committed, storedEvents: [] };
1179 + }),
1180 );
1181 const resolveRuntimePolicy = vi.fn(
1182 ({ thread }: Parameters<RuntimePolicy.RuntimePolicyV2Shape["resolve"]>[0]) =>
@@ -1199,6 +1220,12 @@ function makeLocalCommandHarness(input: {
1220 ),
1221 }),
1222 );
1223 + const getRun = vi.fn<ProjectionStore.ProjectionStoreV2Shape["getRun"]>((_threadId, id) =>
1224 + Effect.as(
1225 + failReadIfRunning,
1226 + projection.runs.find((candidate) => candidate.id === id),
1227 + ),
1228 + );
1229 const layer = ProviderTurnStart.layer.pipe(
1230 Layer.provide(
1231 Layer.mergeAll(
@@ -1250,11 +1277,7 @@ function makeLocalCommandHarness(input: {
1277 getThreadRecords,
1278 getTurnStartContext,
1279 getRuntimeRecoveryProjection,
1253 - getRun: (_threadId, id) =>
1254 - Effect.as(
1255 - failReadIfRunning,
1256 - projection.runs.find((candidate) => candidate.id === id),
1257 - ),
1280 + getRun,
1281 getTurnStartHistory: () =>
1282 input.sideChat !== undefined
1283 ? Effect.succeed([])
@@ -1295,10 +1318,13 @@ function makeLocalCommandHarness(input: {
1318 ),
1319 );
1320 return {
1321 + layer,
1322 + interruptRun,
1323 settledRuns,
1324 getThreadRecords,
1325 getTurnStartContext,
1326 getRuntimeRecoveryProjection,
1327 + getRun,
1328 setProjection: (next: OrchestrationV2ThreadProjection) => {
1329 projection = next;
1330 },
@@ -1330,6 +1356,464 @@ function makeLocalCommandHarness(input: {
1356 };
1357 }
1358
1359 +function layerStartEffectWorker(
1360 + harness: ReturnType<typeof makeLocalCommandHarness>,
1361 + control: Partial<ProviderTurnControlService.ProviderTurnControlServiceV2["Service"]> = {},
1362 +) {
1363 + const executor = EffectWorker.layerExecutor.pipe(
1364 + Layer.provide(
1365 + Layer.mergeAll(
1366 + harness.layer,
1367 + Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({}),
1368 + Layer.mock(ProviderTurnControlService.ProviderTurnControlServiceV2)(control),
1369 + Layer.mock(RunFinalizationService.RunFinalizationService)({}),
1370 + Layer.mock(CheckpointRollbackService.CheckpointRollbackServiceV2)({}),
1371 + Layer.mock(RuntimeRequestService.RuntimeRequestServiceV2)({}),
1372 + Layer.mock(ThreadTitleRegenerationService.ThreadTitleRegenerationService)({}),
1373 + Layer.mock(ThreadManagementService.ThreadManagementService)({}),
1374 + Layer.mock(ProjectionStore.ProjectionStoreV2)({}),
1375 + ServerSettings.layerTest(),
1376 + ),
1377 + ),
1378 + );
1379 + return EffectWorker.layerWithOptions({ maxAttempts: 2 }).pipe(
1380 + Layer.provideMerge(
1381 + Layer.merge(executor, EffectOutbox.layer.pipe(Layer.provide(SqlitePersistence.layerMemory))),
1382 + ),
1383 + );
1384 +}
1385 +
1386 +const enqueueStart = (
1387 + projection: OrchestrationV2ThreadProjection,
1388 + request?: EffectOutbox.OrchestrationEffectRequestV2,
1389 +) =>
1390 + Effect.gen(function* () {
1391 + const outbox = yield* EffectOutbox.EffectOutboxV2;
1392 + const run = projection.runs.at(-1)!;
1393 + yield* outbox.enqueue([
1394 + {
1395 + id: "effect:failed-start",
1396 + commandId: CommandId.make("command:failed-start"),
1397 + threadId: run.threadId,
1398 + request: request ?? { type: "provider-turn.start", runId: run.id },
1399 + },
1400 + ]);
1401 + return { storedEffect: outbox.get("effect:failed-start").pipe(Effect.map(Option.getOrThrow)) };
1402 + });
1403 +
1404 +effectIt.effect(
1405 + "settles an exhausted start effect without claiming an external turn stopped",
1406 + () => {
1407 + const harness = makeLocalCommandHarness({ text: "Continue", missingProviderSession: true });
1408 + const projection = harness.projection();
1409 + const previousAttempt = {
1410 + ...projection.attempts[0]!,
1411 + id: RunAttemptId.make("superseded-attempt"),
1412 + rootNodeId: NodeId.make("superseded-root"),
1413 + status: "superseded" as const,
1414 + };
1415 + const externalTurn = {
1416 + id: ProviderTurnId.make("still-running-provider-turn"),
1417 + providerThreadId: projection.providerThreads[0]!.id,
1418 + nodeId: previousAttempt.rootNodeId,
1419 + runAttemptId: previousAttempt.id,
1420 + nativeTurnRef: null,
1421 + ordinal: 1,
1422 + status: "running" as const,
1423 + startedAt: projection.updatedAt,
1424 + completedAt: null,
1425 + };
1426 + harness.setProjection({
1427 + ...projection,
1428 + attempts: [
1429 + previousAttempt,
1430 + { ...projection.attempts[0]!, attemptOrdinal: 2, reason: "steering_restart" },
1431 + ],
1432 + providerTurns: [externalTurn],
1433 + });
1434 +
1435 + return Effect.gen(function* () {
1436 + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
1437 + const { storedEffect } = yield* enqueueStart(harness.projection());
1438 + expect(yield* worker.drain()).toBe(1);
1439 + expect(yield* storedEffect).toMatchObject({ status: "pending", attemptCount: 1 });
1440 + expect(harness.projection().runs.at(-1)?.status).toBe("starting");
1441 + expect(harness.events).toEqual([]);
1442 +
1443 + yield* TestClock.adjust("100 millis");
1444 + expect(yield* worker.drain()).toBe(1);
1445 + expect(yield* storedEffect).toMatchObject({ status: "succeeded", attemptCount: 2 });
1446 + expect(harness.projection().runs.at(-1)).toMatchObject({ status: "failed", startedAt: null });
1447 + expect(harness.projection().attempts.at(-1)).toMatchObject({
1448 + status: "failed",
1449 + startedAt: null,
1450 + });
1451 + expect(harness.projection().nodes[0]).toMatchObject({ status: "failed", startedAt: null });
1452 + expect(harness.projection().turnItems).toMatchObject([
1453 + {
1454 + type: "error",
1455 + status: "failed",
1456 + title: "Turn failed to start",
1457 + failure: {
1458 + class: "unknown",
1459 + message:
1460 + "T3 Code could not start this turn. Send the message again; if it keeps failing, check the server logs.",
1461 + },
1462 + },
1463 + ]);
1464 + expect(harness.open).not.toHaveBeenCalled();
1465 + expect(harness.startRootRun).not.toHaveBeenCalled();
1466 + expect(harness.projection().providerTurns).toEqual([externalTurn]);
1467 + }).pipe(Effect.provide(layerStartEffectWorker(harness)));
1468 + },
1469 +);
1470 +
1471 +effectIt.effect("retries a failed settlement past the start budget until it is persisted", () => {
1472 + const harness = makeLocalCommandHarness({ text: "Continue", missingProviderSession: true });
1473 + harness.writeIfRunCurrent.mockImplementationOnce(({ events }) =>
1474 + Effect.fail(
1475 + new EventSink.EventSinkWriteError({
1476 + eventCount: events.length,
1477 + cause: "storage unavailable",
1478 + }),
1479 + ),
1480 + );
1481 +
1482 + return Effect.gen(function* () {
1483 + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
1484 + const { storedEffect } = yield* enqueueStart(harness.projection());
1485 + yield* worker.drain();
1486 + yield* TestClock.adjust("100 millis");
1487 + expect(yield* worker.drain()).toBe(1);
1488 + expect(yield* storedEffect).toMatchObject({ status: "pending", attemptCount: 2 });
1489 + expect(harness.projection().runs.at(-1)?.status).toBe("starting");
1490 + expect(harness.events).toEqual([]);
1491 +
1492 + yield* TestClock.adjust("200 millis");
1493 + expect(yield* worker.drain()).toBe(1);
1494 + expect(yield* storedEffect).toMatchObject({ status: "succeeded", attemptCount: 3 });
1495 + expect(harness.projection().runs.at(-1)?.status).toBe("failed");
1496 + expect(harness.projection().turnItems).toHaveLength(1);
1497 + }).pipe(Effect.provide(layerStartEffectWorker(harness)));
1498 +});
1499 +
1500 +effectIt.effect.each(["interrupt", "start"] as const)(
1501 + "keeps a failed restart %s separate from starting-run settlement",
1502 + (stage) => {
1503 + const harness = makeLocalCommandHarness({ text: "Continue", missingProviderSession: true });
1504 + const projection = harness.projection();
1505 + const run = projection.runs.at(-1)!;
1506 + const providerTurnId = ProviderTurnId.make("restart-interrupted-turn");
1507 + return Effect.gen(function* () {
1508 + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
1509 + const { storedEffect } = yield* enqueueStart(projection, {
1510 + type: "provider-turn.restart",
1511 + runId: run.id,
1512 + providerSessionId: ProviderSessionId.make("restart-interrupted-session"),
1513 + providerThreadId: projection.providerThreads[0]!.id,
1514 + providerTurnId,
1515 + interruptedAttemptId: RunAttemptId.make("restart-interrupted-attempt"),
1516 + });
1517 + yield* worker.drain();
1518 + yield* TestClock.adjust("100 millis");
1519 + expect(yield* worker.drain()).toBe(1);
1520 + if (stage === "interrupt") {
1521 + yield* TestClock.adjust("200 millis");
1522 + expect(yield* worker.drain()).toBe(1);
1523 + expect(yield* storedEffect).toMatchObject({ status: "pending", attemptCount: 3 });
1524 + expect(harness.projection().runs.at(-1)?.status).toBe("starting");
1525 + expect(harness.events).toEqual([]);
1526 + expect(harness.writeIfRunCurrent).not.toHaveBeenCalled();
1527 + } else {
1528 + expect(yield* storedEffect).toMatchObject({ status: "succeeded", attemptCount: 2 });
1529 + expect(harness.projection().runs.at(-1)?.status).toBe("failed");
1530 + expect(harness.projection().turnItems).toMatchObject([
1531 + { type: "error", title: "Turn failed to start" },
1532 + ]);
1533 + }
1534 + }).pipe(
1535 + Effect.provide(
1536 + layerStartEffectWorker(harness, {
1537 + interruptAndAwaitTerminal: () =>
1538 + stage === "interrupt"
1539 + ? Effect.fail(
1540 + new ProviderTurnControlService.ProviderTurnControlError({
1541 + threadId: run.threadId,
1542 + operation: "restart",
1543 + providerTurnId,
1544 + cause: "provider interrupt failed",
1545 + }),
1546 + )
1547 + : Effect.void,
1548 + }),
1549 + ),
1550 + );
1551 + },
1552 +);
1553 +
1554 +effectIt.effect.each(["Stop", "a replacement attempt"] as const)(
1555 + "does not overwrite %s committed during the final start failure write",
1556 + (change) => {
1557 + const harness = makeLocalCommandHarness({ text: "Continue", missingProviderSession: true });
1558 + const write = harness.writeIfRunCurrent.getMockImplementation()!;
1559 + let protectedProjection = harness.projection();
1560 + harness.writeIfRunCurrent.mockImplementationOnce((input) =>
1561 + Effect.suspend(() => {
1562 + const projection = harness.projection();
1563 + if (change === "Stop") harness.interruptRun();
1564 + else {
1565 + harness.setProjection({
1566 + ...projection,
1567 + runs: projection.runs.map((run) => ({
1568 + ...run,
1569 + activeAttemptId: RunAttemptId.make("replacement-attempt"),
1570 + rootNodeId: NodeId.make("replacement-root"),
1571 + })),
1572 + });
1573 + }
1574 + protectedProjection = harness.projection();
1575 + return write(input);
1576 + }),
1577 + );
1578 +
1579 + return Effect.gen(function* () {
1580 + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
1581 + const { storedEffect } = yield* enqueueStart(harness.projection());
1582 + yield* worker.drain();
1583 + yield* TestClock.adjust("100 millis");
1584 + expect(yield* worker.drain()).toBe(1);
1585 + expect(harness.writeIfRunCurrent).toHaveBeenCalledOnce();
1586 + expect(harness.projection()).toEqual(protectedProjection);
1587 + expect(harness.events).toEqual([]);
1588 + expect(yield* storedEffect).toMatchObject({ status: "succeeded" });
1589 + }).pipe(Effect.provide(layerStartEffectWorker(harness)));
1590 + },
1591 +);
1592 +
1593 +effectIt.effect("preserves a replacement committed after the final pre-running start read", () => {
1594 + const harness = makeLocalCommandHarness({ text: "Continue", missingProviderSession: true });
1595 + const read = harness.getTurnStartContext.getMockImplementation()!;
1596 + let reads = 0;
1597 + let replacement: OrchestrationV2ThreadProjection;
1598 + harness.getTurnStartContext.mockImplementation(() =>
1599 + Effect.suspend(() => {
1600 + const snapshot = read();
1601 + reads++;
1602 + if (reads === 2) {
1603 + const projection = harness.projection();
1604 + const attemptId = RunAttemptId.make("replacement-before-fallback");
1605 + const rootNodeId = NodeId.make("replacement-root-before-fallback");
1606 + replacement = {
1607 + ...projection,
1608 + runs: projection.runs.map((run) => ({ ...run, activeAttemptId: attemptId, rootNodeId })),
1609 + attempts: [
1610 + ...projection.attempts,
1611 + { ...projection.attempts[0]!, id: attemptId, rootNodeId },
1612 + ],
1613 + nodes: [...projection.nodes, { ...projection.nodes[0]!, id: rootNodeId }],
1614 + };
1615 + harness.setProjection(replacement);
1616 + }
1617 + return snapshot;
1618 + }),
1619 + );
1620 + return Effect.gen(function* () {
1621 + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
1622 + const { storedEffect } = yield* enqueueStart(harness.projection());
1623 + yield* worker.drain();
1624 + yield* TestClock.adjust("100 millis");
1625 + expect(yield* worker.drain()).toBe(1);
1626 + expect(harness.projection()).toEqual(replacement!);
1627 + expect(harness.events).toEqual([]);
1628 + expect(yield* storedEffect).toMatchObject({ status: "succeeded", attemptCount: 2 });
1629 + }).pipe(Effect.provide(layerStartEffectWorker(harness)));
1630 +});
1631 +
1632 +effectIt.effect("preserves a replacement committed before exhausted start settlement reads", () =>
1633 + Effect.gen(function* () {
1634 + const harness = makeLocalCommandHarness({ text: "Continue", failReadsAfterRunning: true });
1635 + harness.getRuntimeRecoveryProjection.mockImplementation(() =>
1636 + Effect.succeed(harness.projection()),
1637 + );
1638 + harness.getRun.mockImplementation((_threadId, id) =>
1639 + Effect.succeed(harness.projection().runs.find((candidate) => candidate.id === id)),
1640 + );
1641 + const projection = harness.projection();
1642 + const run = projection.runs[0]!;
1643 + const replacementAttemptId = RunAttemptId.make("replacement-before-settlement");
1644 + const replacementRootId = NodeId.make("replacement-root-before-settlement");
1645 + const providerTurn = {
1646 + id: ProviderTurnId.make("old-start-provider-turn"),
1647 + providerThreadId: run.providerThreadId!,
1648 + nodeId: run.rootNodeId!,
1649 + runAttemptId: run.activeAttemptId!,
1650 + nativeTurnRef: null,
1651 + ordinal: 1,
1652 + status: "running" as const,
1653 + startedAt: projection.updatedAt,
1654 + completedAt: null,
1655 + };
1656 + const turnIngested = yield* Deferred.make<void>();
1657 + const storageFailure = new EventSink.EventSinkWriteError({
1658 + eventCount: 5,
1659 + cause: "terminal-write ownership query unavailable",
1660 + });
1661 + const write = harness.writeIfRunCurrent.getMockImplementation()!;
1662 + harness.writeIfRunCurrent.mockImplementation((input) =>
1663 + input.expectedStatus === "running" ? Effect.fail(storageFailure) : write(input),
1664 + );
1665 + harness.open.mockImplementationOnce(() =>
1666 + Effect.fail(
1667 + new ProviderSessionManager.ProviderSessionOpenError({
1668 + instanceId: harness.newInstanceId,
1669 + providerSessionId: projection.providerThreads[0]!.providerSessionId!,
1670 + cause: "transient session failure",
1671 + }),
1672 + ),
1673 + );
1674 + const executionLayer = RunExecutionService.layer.pipe(
1675 + Layer.provide(
1676 + Layer.mergeAll(
1677 + McpAppModelContext.layerEmpty,
1678 + PreviewAgentActivity.layer,
1679 + Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }),
1680 + Layer.mock(TurnItemPositionStore.TurnItemPositionStoreV2)({}),
1681 + Layer.mock(EventSink.EventSinkV2)({
1682 + consumeProviderEvents: (source, process, after) =>
1683 + source.pipe(
1684 + Stream.mapEffect((event) => process(event).pipe(Effect.andThen(after))),
1685 + Stream.takeUntil(Boolean),
1686 + Stream.runDrain,
1687 + ),
1688 + writeIfRunCurrent: harness.writeIfRunCurrent,
1689 + }),
1690 + IdAllocator.layer,
1691 + ThreadCommandExecutor.layer,
1692 + Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
1693 + ingestNormalized: () =>
1694 + Effect.sync(() => {
1695 + harness.setProjection({ ...harness.projection(), providerTurns: [providerTurn] });
1696 + }).pipe(Effect.andThen(Deferred.succeed(turnIngested, undefined)), Effect.as([])),
1697 + }),
1698 + ServerSettings.layerTest(),
1699 + ),
1700 + ),
1701 + );
1702 + let protectedProjection = projection;
1703 + let escapedStartError: RunExecutionService.RunExecutionServiceV2Error | undefined;
1704 + return yield* Effect.gen(function* () {
1705 + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
1706 + const outbox = yield* EffectOutbox.EffectOutboxV2;
1707 + harness.startRootRun.mockImplementation((input) =>
1708 + Effect.gen(function* () {
1709 + const execution = yield* RunExecutionService.RunExecutionServiceV2;
1710 + yield* execution.startRootRun({
1711 + ...input,
1712 + session: {
1713 + ...input.session,
1714 + events: Stream.concat(
1715 + Stream.succeed({
1716 + type: "provider_turn.updated" as const,
1717 + driver: input.providerThread.driver,
1718 + threadId: run.threadId,
1719 + providerTurn,
1720 + }),
1721 + Stream.never,
1722 + ),
1723 + startTurn: () =>
1724 + Effect.gen(function* () {
1725 + yield* Deferred.await(turnIngested);
1726 + const current = harness.projection();
1727 + expect(current.runs[0]?.status).toBe("running");
1728 + // Steering commits a new attempt while the original start still owns the lane.
1729 + harness.setProjection({
1730 + ...current,
1731 + runs: [
1732 + {
1733 + ...current.runs[0]!,
1734 + status: "starting",
1735 + activeAttemptId: replacementAttemptId,
1736 + rootNodeId: replacementRootId,
1737 + },
1738 + ],
1739 + attempts: [
1740 + {
1741 + ...current.attempts[0]!,
1742 + status: "superseded",
1743 + completedAt: current.updatedAt,
1744 + },
1745 + {
1746 + ...projection.attempts[0]!,
1747 + id: replacementAttemptId,
1748 + rootNodeId: replacementRootId,
1749 + attemptOrdinal: 2,
1750 + reason: "steering_restart",
1751 + },
1752 + ],
1753 + nodes: [
1754 + {
1755 + ...current.nodes[0]!,
1756 + status: "interrupted",
1757 + completedAt: current.updatedAt,
1758 + },
1759 + {
1760 + ...projection.nodes[0]!,
1761 + id: replacementRootId,
1762 + rootNodeId: replacementRootId,
1763 + },
1764 + ],
1765 + });
1766 + protectedProjection = harness.projection();
1767 + yield* outbox
1768 + .enqueue([
1769 + {
1770 + id: "effect:replacement-restart",
1771 + commandId: CommandId.make("command:replacement-restart"),
1772 + threadId: run.threadId,
1773 + request: {
1774 + type: "provider-turn.restart",
1775 + runId: run.id,
1776 + providerSessionId: input.providerSessionId,
1777 + providerThreadId: run.providerThreadId!,
1778 + providerTurnId: providerTurn.id,
1779 + interruptedAttemptId: input.attemptId,
1780 + },
1781 + },
1782 + ])
1783 + .pipe(Effect.orDie);
1784 + return yield* new ProviderAdapter.ProviderAdapterTurnStartError({
1785 + driver: input.session.driver,
1786 + threadId: run.threadId,
1787 + providerThreadId: run.providerThreadId!,
1788 + runId: run.id,
1789 + cause: "provider start failed after publishing its turn",
1790 + });
1791 + }),
1792 + },
1793 + });
1794 + }).pipe(
1795 + Effect.tapError((error) =>
1796 + Effect.sync(() => {
1797 + escapedStartError = error;
1798 + }),
1799 + ),
1800 + Effect.provide(executionLayer.pipe(Layer.orDie)),
1801 + ),
1802 + );
1803 + const { storedEffect } = yield* enqueueStart(projection);
1804 + expect(yield* worker.drain()).toBe(1);
1805 + yield* TestClock.adjust("100 millis");
1806 + expect(yield* worker.drain(1)).toBe(1);
1807 + expect(escapedStartError).toMatchObject({ cause: { write: storageFailure } });
1808 + expect(yield* outbox.get("effect:replacement-restart")).toMatchObject({
1809 + value: { status: "pending", attemptCount: 0 },
1810 + });
1811 + expect(yield* storedEffect).toMatchObject({ status: "succeeded", attemptCount: 2 });
1812 + expect(harness.projection()).toEqual(protectedProjection);
1813 + }).pipe(Effect.provide(layerStartEffectWorker(harness)));
1814 + }),
1815 +);
1816 +
1817 effectIt.effect(
1818 "recovers the local or pushed thread tip, else the current branch, remote default, or detached HEAD",
1819 () =>
apps/server/src/orchestration-v2/ProviderTurnStartService.ts
+241 -118
@@ -13,8 +13,11 @@ import {
13 isCompactionCommandMessage,
14 parseGoalCommand,
15 RunId,
16 + RunAttemptId,
17 + NodeId,
18 ThreadId,
19 } from "@t3tools/contracts";
20 +import * as Cause from "effect/Cause";
21 import * as Context from "effect/Context";
22 import * as DateTime from "effect/DateTime";
23 import * as Effect from "effect/Effect";
@@ -75,6 +78,8 @@ export class ProviderTurnStartError extends Schema.TaggedError<ProviderTurnStart
78 {
79 runId: RunId,
80 cause: Schema.optional(Schema.Defect()),
81 + attemptId: Schema.optional(RunAttemptId),
82 + rootNodeId: Schema.optional(NodeId),
83 },
84 ) {}
85
@@ -86,7 +91,7 @@ const NON_COMPLETING_TERMINAL_RUN_STATUSES: ReadonlySet<OrchestrationV2Run["stat
91 "rolled_back",
92 ]);
93
89 -const isProviderTurnStartError = Schema.is(ProviderTurnStartError);
94 +export const isProviderTurnStartError = Schema.is(ProviderTurnStartError);
95 const isProviderAdapterProtocolError = Schema.is(ProviderAdapter.ProviderAdapterProtocolError);
96 const isProviderAdapterOpenSessionError = Schema.is(
97 ProviderAdapter.ProviderAdapterOpenSessionError,
@@ -113,6 +118,21 @@ const refusedBeforePrompt = (error: unknown): boolean =>
118 error.detail === CLAUDE_QUERY_ENDED_BEFORE_TURN_DETAIL) ||
119 (Predicate.hasProperty(error, "cause") && refusedBeforePrompt(error.cause));
120
121 +/** Missing execution records cannot be repaired by retrying a start. */
122 +export class ProviderTurnStartRunStateMissingError extends Schema.TaggedError<ProviderTurnStartRunStateMissingError>()(
123 + "ProviderTurnStartRunStateMissingError",
124 + {
125 + runId: RunId,
126 + },
127 +) {}
128 +
129 +export const isProviderTurnStartRunStateMissingError = Schema.is(
130 + ProviderTurnStartRunStateMissingError,
131 +);
132 +
133 +const START_GAVE_UP_MESSAGE =
134 + "T3 Code could not start this turn. Send the message again; if it keeps failing, check the server logs.";
135 +
136 const startFailureMessage = (error: Error): string => {
137 const nestedCause = "cause" in error ? error.cause : undefined;
138 return (
@@ -131,6 +151,12 @@ export interface ProviderTurnStartServiceV2Shape {
151 readonly runId: RunId;
152 readonly willRetry?: boolean;
153 }) => Effect.Effect<void, ProviderTurnStartError>;
154 + readonly failStartingRun: (input: {
155 + readonly threadId: ThreadId;
156 + readonly runId: RunId;
157 + readonly attemptId: RunAttemptId;
158 + readonly rootNodeId: NodeId;
159 + }) => Effect.Effect<void, ProviderTurnStartError | ProviderTurnStartRunStateMissingError>;
160 }
161
162 export class ProviderTurnStartServiceV2 extends Context.Service<
@@ -289,11 +315,109 @@ export const layer: Layer.Layer<
315 };
316 };
317
292 - const start = Effect.fn("orchestrationV2.providerTurnStart.start")(function* (input: {
293 - readonly threadId: ThreadId;
294 - readonly runId: RunId;
295 - readonly willRetry?: boolean;
296 - }) {
318 + const settleRunBeforeStart = Effect.fn("orchestrationV2.providerTurnStart.settleBeforeStart")(
319 + function* (input: {
320 + readonly run: OrchestrationV2Run;
321 + readonly rootNode: OrchestrationV2ExecutionNode;
322 + readonly attempt: OrchestrationV2RunAttempt;
323 + readonly turnItems: ReadonlyArray<OrchestrationV2TurnItem>;
324 + readonly signal: string;
325 + readonly status: "completed" | "failed";
326 + readonly now: DateTime.Utc;
327 + readonly startedAt?: DateTime.Utc;
328 + readonly providerInstanceId: OrchestrationV2Run["providerInstanceId"];
329 + readonly itemProviderThreadId: OrchestrationV2ProviderThread["id"] | null;
330 + readonly item:
331 + | Pick<Extract<OrchestrationV2TurnItem, { type: "error" }>, "type" | "title" | "failure">
332 + | Pick<
333 + Extract<OrchestrationV2TurnItem, { type: "command_execution" }>,
334 + "type" | "title" | "input" | "output" | "exitCode"
335 + >;
336 + readonly providerThreadUpdate?: OrchestrationV2ProviderThread;
337 + }) {
338 + const { run, rootNode, attempt, turnItems, now, status } = input;
339 + const runId = run.id;
340 + const started = input.startedAt === undefined ? {} : { startedAt: input.startedAt };
341 + const item: OrchestrationV2TurnItem = {
342 + id: idAllocator.derive.runSignalTurnItem({ runId, signal: input.signal }),
343 + threadId: run.threadId,
344 + runId,
345 + nodeId: rootNode.id,
346 + providerThreadId: input.itemProviderThreadId,
347 + providerTurnId: null,
348 + nativeItemRef: null,
349 + parentItemId: null,
350 + ordinal:
351 + Math.max(
352 + 0,
353 + ...turnItems.filter((item) => item.runId === runId).map((item) => item.ordinal),
354 + ) + 1,
355 + status,
356 + startedAt: now,
357 + completedAt: now,
358 + updatedAt: now,
359 + ...input.item,
360 + };
361 + const eventPayloads = [
362 + { type: "turn-item.updated", payload: item },
363 + { type: "run.updated", payload: { ...run, status, ...started, completedAt: now } },
364 + {
365 + type: "run-attempt.updated",
366 + payload: { ...attempt, status, ...started, completedAt: now },
367 + },
368 + {
369 + type: "node.updated",
370 + payload: { ...rootNode, status, ...started, completedAt: now },
371 + },
372 + ...(input.providerThreadUpdate === undefined
373 + ? []
374 + : [
375 + {
376 + type: "provider-thread.updated" as const,
377 + payload: input.providerThreadUpdate,
378 + },
379 + ]),
380 + ] as const;
381 + const events = yield* Effect.forEach(eventPayloads, (event) =>
382 + Effect.gen(function* () {
383 + return {
384 + ...event,
385 + id: yield* idAllocator.allocate.event({ threadId: run.threadId }),
386 + threadId: run.threadId,
387 + runId,
388 + nodeId: rootNode.id,
389 + providerInstanceId: input.providerInstanceId,
390 + occurredAt: now,
391 + } satisfies OrchestrationV2DomainEvent;
392 + }),
393 + );
394 + const written = yield* eventSink.writeIfRunCurrent({
395 + threadId: run.threadId,
396 + runId,
397 + activeAttemptId: attempt.id,
398 + expectedStatus: "starting",
399 + events,
400 + });
401 + // A steering restart that cannot start still ends a run whose earlier turns counted.
402 + if (written.committed) {
403 + yield* turnAnalytics.runSettled({
404 + runId,
405 + status,
406 + completedAt: now,
407 + childrenRunning: false,
408 + });
409 + }
410 + },
411 + );
412 +
413 + const start = Effect.fn("orchestrationV2.providerTurnStart.start")(function* (
414 + input: {
415 + readonly threadId: ThreadId;
416 + readonly runId: RunId;
417 + readonly willRetry?: boolean;
418 + },
419 + captureStartingRun: (run: OrchestrationV2Run) => void,
420 + ) {
421 const { runId } = input;
422 const projection = yield* projectionStore.getTurnStartContext(input.threadId, runId);
423 const run = projection.runs.find((candidate) => candidate.id === runId);
@@ -303,6 +427,7 @@ export const layer: Layer.Layer<
427 if (run.status !== "starting") {
428 return;
429 }
430 + captureStartingRun(run);
431 const rootNode = projection.nodes.find((candidate) => candidate.id === run.rootNodeId);
432 const attempt = projection.attempts.find((candidate) => candidate.id === run.activeAttemptId);
433 const providerThread = projection.providerThreads.find(
@@ -385,100 +510,6 @@ export const layer: Layer.Layer<
510 ? projection.messages.find((source) => source.id === resumeSource.userMessageId)
511 : undefined;
512 const providerMessage = unsentResumeMessage ?? message;
388 - const settleRunBeforeStart = Effect.fn("orchestrationV2.providerTurnStart.settleBeforeStart")(
389 - function* (input: {
390 - readonly signal: string;
391 - readonly status: "completed" | "failed";
392 - readonly now: DateTime.Utc;
393 - readonly startedAt?: DateTime.Utc;
394 - readonly providerInstanceId: OrchestrationV2Run["providerInstanceId"];
395 - readonly itemProviderThreadId: OrchestrationV2ProviderThread["id"];
396 - readonly item:
397 - | Pick<
398 - Extract<OrchestrationV2TurnItem, { type: "error" }>,
399 - "type" | "title" | "failure"
400 - >
401 - | Pick<
402 - Extract<OrchestrationV2TurnItem, { type: "command_execution" }>,
403 - "type" | "title" | "input" | "output" | "exitCode"
404 - >;
405 - readonly providerThreadUpdate?: OrchestrationV2ProviderThread;
406 - }) {
407 - const { now, status } = input;
408 - const started = input.startedAt === undefined ? {} : { startedAt: input.startedAt };
409 - const item: OrchestrationV2TurnItem = {
410 - id: idAllocator.derive.runSignalTurnItem({ runId, signal: input.signal }),
411 - threadId: projection.thread.id,
412 - runId,
413 - nodeId: rootNode.id,
414 - providerThreadId: input.itemProviderThreadId,
415 - providerTurnId: null,
416 - nativeItemRef: null,
417 - parentItemId: null,
418 - ordinal:
419 - Math.max(
420 - 0,
421 - ...projection.turnItems
422 - .filter((item) => item.runId === runId)
423 - .map((item) => item.ordinal),
424 - ) + 1,
425 - status,
426 - startedAt: now,
427 - completedAt: now,
428 - updatedAt: now,
429 - ...input.item,
430 - };
431 - const eventPayloads = [
432 - { type: "turn-item.updated", payload: item },
433 - { type: "run.updated", payload: { ...run, status, ...started, completedAt: now } },
434 - {
435 - type: "run-attempt.updated",
436 - payload: { ...attempt, status, ...started, completedAt: now },
437 - },
438 - {
439 - type: "node.updated",
440 - payload: { ...rootNode, status, ...started, completedAt: now },
441 - },
442 - ...(input.providerThreadUpdate === undefined
443 - ? []
444 - : [
445 - {
446 - type: "provider-thread.updated" as const,
447 - payload: input.providerThreadUpdate,
448 - },
449 - ]),
450 - ] as const;
451 - const events = yield* Effect.forEach(eventPayloads, (event) =>
452 - Effect.gen(function* () {
453 - return {
454 - ...event,
455 - id: yield* idAllocator.allocate.event({ threadId: projection.thread.id }),
456 - threadId: projection.thread.id,
457 - runId,
458 - nodeId: rootNode.id,
459 - providerInstanceId: input.providerInstanceId,
460 - occurredAt: now,
461 - } satisfies OrchestrationV2DomainEvent;
462 - }),
463 - );
464 - const written = yield* eventSink.writeIfRunCurrent({
465 - threadId: projection.thread.id,
466 - runId,
467 - activeAttemptId: attempt.id,
468 - expectedStatus: "starting",
469 - events,
470 - });
471 - // A steering restart that cannot start still ends a run whose earlier turns counted.
472 - if (written.committed) {
473 - yield* turnAnalytics.runSettled({
474 - runId,
475 - status,
476 - completedAt: now,
477 - childrenRunning: false,
478 - });
479 - }
480 - },
481 - );
513 const settleStartFailure = Effect.fn("ProviderTurnStartService.settleStartFailure")(
514 function* (failed: {
515 readonly signal: string;
@@ -487,6 +518,10 @@ export const layer: Layer.Layer<
518 readonly message?: string;
519 }) {
520 yield* settleRunBeforeStart({
521 + run,
522 + rootNode,
523 + attempt,
524 + turnItems: projection.turnItems,
525 signal: failed.signal,
526 status: "failed",
527 now: yield* DateTime.now,
@@ -561,6 +596,10 @@ export const layer: Layer.Layer<
596 : undefined;
597 const status = failure === undefined ? "completed" : "failed";
598 yield* settleRunBeforeStart({
599 + run,
600 + rootNode,
601 + attempt,
602 + turnItems: projection.turnItems,
603 signal: isEmptyCompaction ? "empty-compaction" : "provider-sign-out",
604 status,
605 now,
@@ -1833,25 +1872,65 @@ export const layer: Layer.Layer<
1872 return ProviderTurnStartServiceV2.of({
1873 start: Effect.fn("ProviderTurnStartService.start")(
1874 function* (input) {
1836 - // Storage cleanup removes worktrees under the same lease. Only startup
1837 - // holds it; the provider turn runs after it is released.
1838 - const { thread } = yield* projectionStore.getThreadRecords(input.threadId, []);
1839 - const cwd =
1840 - thread.worktreePath ??
1841 - (yield* projects
1842 - .getById(thread.projectId, { includeDeleted: true })
1843 - .pipe(Effect.map((project) => Option.getOrUndefined(project)?.workspaceRoot)));
1844 - const providerTurn = yield* cwd === undefined
1845 - ? start(input)
1846 - : withTurnStartReservation(
1847 - fileSystem,
1848 - path,
1849 - path.resolve(cwd),
1850 - thread.worktreePath === null
1851 - ? start(input)
1852 - : withWorkspaceLease(path.resolve(thread.worktreePath), start(input)),
1875 + let startingRun: OrchestrationV2Run | undefined;
1876 + const captureStartingRun = (run: OrchestrationV2Run) => {
1877 + startingRun = run;
1878 + };
1879 + return yield* Effect.gen(function* () {
1880 + // Storage cleanup removes worktrees under the same lease. Only startup
1881 + // holds it; the provider turn runs after it is released.
1882 + const { thread } = yield* projectionStore.getThreadRecords(input.threadId, []);
1883 + const cwd =
1884 + thread.worktreePath ??
1885 + (yield* projects
1886 + .getById(thread.projectId, { includeDeleted: true })
1887 + .pipe(Effect.map((project) => Option.getOrUndefined(project)?.workspaceRoot)));
1888 + const providerTurn = yield* cwd === undefined
1889 + ? start(input, captureStartingRun)
1890 + : withTurnStartReservation(
1891 + fileSystem,
1892 + path,
1893 + path.resolve(cwd),
1894 + thread.worktreePath === null
1895 + ? start(input, captureStartingRun)
1896 + : withWorkspaceLease(
1897 + path.resolve(thread.worktreePath),
1898 + start(input, captureStartingRun),
1899 + ),
1900 + );
1901 + if (providerTurn !== undefined) {
1902 + // Run execution owns finalization after the running write. A later
1903 + // failure must not settle a replacement attempt committed by steering.
1904 + yield* providerTurn.pipe(
1905 + Effect.catchCauseIf(
1906 + (cause) => !Cause.hasInterruptsOnly(cause),
1907 + (cause) =>
1908 + Effect.logWarning(
1909 + "Run execution failed after starting",
1910 + { runId: input.runId },
1911 + cause,
1912 + ),
1913 + ),
1914 );
1854 - if (providerTurn !== undefined) yield* providerTurn;
1915 + }
1916 + }).pipe(
1917 + Effect.catchCauseIf(
1918 + (cause) => !Cause.hasInterruptsOnly(cause),
1919 + (cause) =>
1920 + Effect.fail(
1921 + new ProviderTurnStartError({
1922 + runId: input.runId,
1923 + cause: Cause.squash(cause),
1924 + ...(startingRun?.activeAttemptId == null || startingRun.rootNodeId == null
1925 + ? {}
1926 + : {
1927 + attemptId: startingRun.activeAttemptId,
1928 + rootNodeId: startingRun.rootNodeId,
1929 + }),
1930 + }),
1931 + ),
1932 + ),
1933 + );
1934 },
1935 (effect, input) =>
1936 effect.pipe(
@@ -1862,6 +1941,50 @@ export const layer: Layer.Layer<
1941 ),
1942 ),
1943 ),
1944 + failStartingRun: Effect.fn("ProviderTurnStartService.failStartingRun")(
1945 + function* ({ threadId, runId, attemptId, rootNodeId }) {
1946 + const projection = yield* projectionStore.getTurnStartContext(threadId, runId);
1947 + const run = projection.runs.find((candidate) => candidate.id === runId);
1948 + if (
1949 + run === undefined ||
1950 + run.status !== "starting" ||
1951 + run.activeAttemptId !== attemptId ||
1952 + run.rootNodeId !== rootNodeId
1953 + )
1954 + return;
1955 + const rootNode = projection.nodes.find((candidate) => candidate.id === run.rootNodeId);
1956 + const attempt = projection.attempts.find(
1957 + (candidate) => candidate.id === run.activeAttemptId,
1958 + );
1959 + if (rootNode === undefined || attempt === undefined) {
1960 + return yield* new ProviderTurnStartRunStateMissingError({ runId });
1961 + }
1962 + yield* settleRunBeforeStart({
1963 + run,
1964 + rootNode,
1965 + attempt,
1966 + turnItems: projection.turnItems,
1967 + signal: "provider-turn-start-failure",
1968 + status: "failed",
1969 + now: yield* DateTime.now,
1970 + providerInstanceId: run.providerInstanceId,
1971 + itemProviderThreadId: run.providerThreadId,
1972 + item: {
1973 + type: "error",
1974 + title: "Turn failed to start",
1975 + failure: makeProviderFailure({ message: START_GAVE_UP_MESSAGE }),
1976 + },
1977 + });
1978 + },
1979 + (effect, { runId }) =>
1980 + effect.pipe(
1981 + Effect.mapError((cause) =>
1982 + isProviderTurnStartRunStateMissingError(cause)
1983 + ? cause
1984 + : new ProviderTurnStartError({ runId, cause }),
1985 + ),
1986 + ),
1987 + ),
1988 });
1989 }),
1990 );