The best way to control Claude Code, Codex and more from any computer or phone

main
ts 3,408 lines 127 KB
Raw
1 import {
2 serializeAssistantCitation,
3 expandAssistantCitationsForProvider,
4 } from "@t3tools/shared/assistantCitations";
5 import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
6 import * as NodeServices from "@effect/platform-node/NodeServices";
7 import { expect, it, vi } from "vite-plus/test";
8 import { it as effectIt } from "@effect/vitest";
9 import {
10 CheckpointScopeId,
11 CommandId,
12 ContextHandoffId,
13 ContextTransferId,
14 GitCommandError,
15 MessageId,
16 NodeId,
17 ProviderSessionId,
18 ProviderThreadId,
19 ProviderTurnId,
20 ProviderDriverKind,
21 ProviderInstanceId,
22 ProviderSetupError,
23 RunAttemptId,
24 RunId,
25 ScheduledTaskId,
26 ThreadId,
27 TurnItemId,
28 ProjectId,
29 EnvironmentId,
30 PROVIDER_SEND_TURN_MAX_INPUT_CHARS,
31 type OrchestrationV2ThreadProjection,
32 OrchestrationV2DomainEvent,
33 } from "@t3tools/contracts";
34 import * as Cause from "effect/Cause";
35 import * as DateTime from "effect/DateTime";
36 import * as Deferred from "effect/Deferred";
37 import * as Effect from "effect/Effect";
38 import * as FileSystem from "effect/FileSystem";
39 import * as Fiber from "effect/Fiber";
40 import * as Layer from "effect/Layer";
41 import * as Option from "effect/Option";
42 import * as Path from "effect/Path";
43 import * as PlatformError from "effect/PlatformError";
44 import * as Schema from "effect/Schema";
45 import * as Stream from "effect/Stream";
46 import * as TestClock from "effect/testing/TestClock";
47
48 import * as ServerConfig from "../config.ts";
49 import * as McpAppModelContext from "../mcpApps/McpAppModelContext.ts";
50 import * as PreviewAgentActivity from "../preview/PreviewAgentActivity.ts";
51 import * as CheckpointService from "./CheckpointService.ts";
52 import * as ProviderEventIngestor from "./ProviderEventIngestor.ts";
53 import {
54 withCheckpointRestoreReservation,
55 withWorkspaceLease,
56 } from "../workspace/workspaceLease.ts";
57 import * as GitWorkflow from "../git/GitWorkflowService.ts";
58 import {
59 layerThreadWorktreeRestore,
60 ThreadWorktreeRestore,
61 } from "../workspace/restoreMissingWorktree.ts";
62 import * as GitVcsDriver from "../vcs/GitVcsDriver.ts";
63 import { layerRealGit } from "./ProviderTurnStartService.testkit.ts";
64 import * as ProjectService from "../project/ProjectService.ts";
65 import * as ProviderAuthService from "../provider/ProviderAuthService.ts";
66 import * as ServerSettings from "../serverSettings.ts";
67 import * as ContextHandoffService from "./ContextHandoffService.ts";
68 import { withInlineContext } from "./ContextHandoffDelivery.ts";
69 import * as EventSink from "./EventSink.ts";
70 import * as SqlitePersistence from "../persistence/Sqlite.ts";
71 import * as EffectOutbox from "./EffectOutbox.ts";
72 import * as EffectWorker from "./EffectWorker.ts";
73 import * as CheckpointRollbackService from "./CheckpointRollbackService.ts";
74 import * as ProviderTurnControlService from "./ProviderTurnControlService.ts";
75 import * as RunFinalizationService from "./RunFinalizationService.ts";
76 import * as RuntimeRequestService from "./RuntimeRequestService.ts";
77 import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
78 import * as ThreadManagementService from "./ThreadManagementService.ts";
79 import * as ThreadTitleRegenerationService from "./ThreadTitleRegenerationService.ts";
80 import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
81 import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
82 import * as ProjectionStore from "./ProjectionStore.ts";
83 import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
84 import * as ProviderSessionManager from "./ProviderSessionManager.ts";
85 import * as ProviderTurnStart from "./ProviderTurnStartService.ts";
86 import * as RunExecutionService from "./RunExecutionService.ts";
87 import * as ProviderUsageAnalytics from "../telemetry/ProviderUsageAnalytics.ts";
88 import * as RuntimePolicy from "./RuntimePolicy.ts";
89 import * as TurnItemPositionStore from "./TurnItemPositionStore.ts";
90
91 const isDomainEvent = Schema.is(OrchestrationV2DomainEvent);
92
93 const managedWorktreesDir = "/tmp/provider-turn-start-worktrees";
94 // fs.realpath applies `..` and `.` before following symlinks.
95 const normalize = (target: string) => {
96 const parts: Array<string> = [];
97 for (const part of target.split("/")) {
98 if (part === "..") parts.pop();
99 else if (part !== "" && part !== ".") parts.push(part);
100 }
101 return `/${parts.join("/")}`;
102 };
103 const layerWorktreeLocation = (worktreesDir = managedWorktreesDir) =>
104 Layer.mergeAll(
105 Path.layer,
106 Layer.mock(ServerConfig.ServerConfig)({ worktreesDir } as ServerConfig.ServerConfig["Service"]),
107 ServerSettings.layerTest(),
108 );
109
110 function startWithMissingWorktree(
111 worktreePath: string,
112 realPaths: Record<string, string> = {
113 [managedWorktreesDir]: managedWorktreesDir,
114 [`${managedWorktreesDir}/project`]: `${managedWorktreesDir}/project`,
115 },
116 ) {
117 const threadId = ThreadId.make("thread_provider_turn_start_projection_failure");
118 const runId = RunId.make("run_provider_turn_start_projection_failure");
119 const attemptId = RunAttemptId.make("attempt_provider_turn_start_projection_failure");
120 const rootNodeId = NodeId.make("node_provider_turn_start_projection_failure");
121 const providerThreadId = ProviderThreadId.make(
122 "provider_thread_provider_turn_start_projection_failure",
123 );
124 const providerSessionId = ProviderSessionId.make(
125 "provider_session_provider_turn_start_projection_failure",
126 );
127 const messageId = MessageId.make("message_provider_turn_start_projection_failure");
128 const checkpointScopeId = CheckpointScopeId.make(
129 "checkpoint_scope_provider_turn_start_projection_failure",
130 );
131 const projection = {
132 thread: {
133 id: threadId,
134 projectId: ProjectId.make("project_provider_turn_start_projection_failure"),
135 branch: "feature/restore",
136 worktreePath,
137 lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId },
138 },
139 runs: [
140 {
141 id: runId,
142 status: "starting",
143 rootNodeId,
144 activeAttemptId: attemptId,
145 providerThreadId,
146 userMessageId: messageId,
147 ordinal: 2,
148 },
149 ],
150 nodes: [{ id: rootNodeId, checkpointScopeId }],
151 attempts: [{ id: attemptId }],
152 providerThreads: [{ id: providerThreadId, providerSessionId }],
153 messages: [{ id: messageId, text: "Continue", attachments: [] }],
154 checkpointScopes: [{ id: checkpointScopeId }],
155 contextHandoffs: [],
156 contextTransfers: [],
157 turnItems: [],
158 } as unknown as OrchestrationV2ThreadProjection;
159 let projectionReadCount = 0;
160 const writeIfRunCurrent = vi.fn(() =>
161 Effect.succeed({ committed: true, storedEvents: [] } as never),
162 );
163 const startRootRun = vi.fn(() => Effect.void);
164 const pruneWorktrees = vi.fn(() => Effect.void);
165 const createWorktree = vi.fn(() => Effect.succeed({} as never));
166 const listLocalBranchNames = vi.fn(() => Effect.succeed(["feature/restore"]));
167 const layer = ProviderTurnStart.layer.pipe(
168 Layer.provide(
169 Layer.mergeAll(
170 Layer.mock(ContextHandoffService.ContextHandoffServiceV2)({}),
171 Layer.mock(EventSink.EventSinkV2)({ writeIfRunCurrent }),
172 IdAllocator.layer,
173 Layer.mock(TurnItemPositionStore.TurnItemPositionStoreV2)({}),
174 Layer.succeed(FileSystem.FileSystem, {
175 exists: () => Effect.succeed(false),
176 realPath: (target: string) =>
177 normalize(target) in realPaths
178 ? Effect.succeed(realPaths[normalize(target)])
179 : Effect.fail(
180 PlatformError.systemError({
181 _tag: "NotFound",
182 module: "FileSystem",
183 method: "realPath",
184 pathOrDescriptor: target,
185 }),
186 ),
187 } as never),
188 layerWorktreeLocation(),
189 Layer.mock(GitWorkflow.GitWorkflowService)({
190 pruneWorktrees,
191 createWorktree,
192 listLocalBranchNames,
193 }),
194 Layer.mock(ProjectService.ProjectService)({
195 getById: () =>
196 Effect.succeedSome({ workspaceRoot: "/tmp/provider-turn-start-project" } as never),
197 }),
198 Layer.mock(ProjectionStore.ProjectionStoreV2)({
199 getThreadRecords: () => {
200 projectionReadCount += 1;
201 return Effect.succeed(projection);
202 },
203 getTurnStartContext: () => {
204 projectionReadCount += 1;
205 return Effect.succeed({
206 ...projection,
207 hasConversation: projection.messages.some(
208 (m) =>
209 m.role === "user" &&
210 (m.text.trim().toLowerCase() !== "/compact" || m.attachments.length > 0),
211 ),
212 });
213 },
214 getRuntimeRecoveryProjection: () => {
215 projectionReadCount += 1;
216 return Effect.fail(
217 new ProjectionStore.ProjectionStoreReadError({
218 threadId,
219 cause: "simulated inherited-background projection failure",
220 }),
221 );
222 },
223 }),
224 Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({}),
225 Layer.mock(ProviderAuthService.ProviderAuthService)({
226 tryHandlePromptCommand: () => Effect.succeed(false),
227 }),
228 Layer.mock(RunExecutionService.RunExecutionServiceV2)({ startRootRun }),
229 Layer.mock(RuntimePolicy.RuntimePolicyV2)({}),
230 ),
231 ),
232 );
233
234 const start = Effect.gen(function* () {
235 return yield* (yield* ProviderTurnStart.ProviderTurnStartServiceV2)
236 .start({ threadId, runId, willRetry: true })
237 .pipe(Effect.flip);
238 }).pipe(Effect.provide(layer), Effect.runPromise);
239 return {
240 start,
241 pruneWorktrees,
242 createWorktree,
243 writeIfRunCurrent,
244 startRootRun,
245 projectionReads: () => projectionReadCount,
246 };
247 }
248
249 it.each([
250 {
251 label: "<managed>/<repo>/<branch>",
252 worktreePath: `${managedWorktreesDir}/project/feature-restore`,
253 recreated: true,
254 },
255 {
256 label: "the real path of a symlinked managed folder",
257 worktreePath: "/real-worktrees/project/feature-restore",
258 realPaths: { [managedWorktreesDir]: "/real-worktrees", "/real-worktrees": "/real-worktrees" },
259 recreated: true,
260 },
261 {
262 label: "a path outside the managed folder",
263 worktreePath: "/tmp/provider-turn-start-project/.git/hooks",
264 recreated: false,
265 },
266 {
267 label: "a symlink inside the managed folder that leads out",
268 worktreePath: `${managedWorktreesDir}/project/hooks`,
269 realPaths: {
270 [managedWorktreesDir]: managedWorktreesDir,
271 [`${managedWorktreesDir}/project`]: "/tmp/provider-turn-start-project/.git",
272 },
273 recreated: false,
274 },
275 {
276 label: "a folder nested below a worktree",
277 worktreePath: `${managedWorktreesDir}/project/feature-restore/link/hooks`,
278 recreated: false,
279 },
280 {
281 label: "'..' out of the managed folder",
282 worktreePath: `${managedWorktreesDir}/project/../../outside`,
283 recreated: false,
284 },
285 {
286 // Left raw on purpose: git follows `link` before applying `..`.
287 label: "'..' after a symlink inside the managed folder",
288 worktreePath: `${managedWorktreesDir}/project/x/link/../../hooks`,
289 realPaths: {
290 [managedWorktreesDir]: managedWorktreesDir,
291 [`${managedWorktreesDir}/project`]: `${managedWorktreesDir}/project`,
292 [`${managedWorktreesDir}/project/x`]: `${managedWorktreesDir}/project/x`,
293 [`${managedWorktreesDir}/project/x/link`]: "/tmp/provider-turn-start-project/.git/refs/heads",
294 },
295 recreated: false,
296 },
297 { label: "a relative path", worktreePath: "project/feature-restore", recreated: false },
298 ])("recreates a missing worktree only at the managed location: $label", async (input) => {
299 const harness = startWithMissingWorktree(input.worktreePath, input.realPaths);
300 await harness.start;
301 expect(harness.createWorktree).toHaveBeenCalledTimes(input.recreated ? 1 : 0);
302 });
303
304 const restoreLeaseExists = vi.fn(() => Effect.succeed(false));
305 const restoreLeaseCreateWorktree = vi.fn(() => Effect.succeed({} as never));
306 const layerRestoreLeaseTest = layerThreadWorktreeRestore.pipe(
307 Layer.provide(
308 Layer.mergeAll(
309 Layer.succeed(FileSystem.FileSystem, {
310 exists: restoreLeaseExists,
311 realPath: (target: string) => Effect.succeed(normalize(target)),
312 } as never),
313 layerWorktreeLocation(),
314 Layer.mock(GitWorkflow.GitWorkflowService)({
315 pruneWorktrees: () => Effect.void,
316 createWorktree: restoreLeaseCreateWorktree,
317 listLocalBranchNames: () => Effect.succeed(["feature/restore"]),
318 }),
319 Layer.mock(ProjectService.ProjectService)({
320 getById: () =>
321 Effect.succeedSome({ workspaceRoot: "/tmp/provider-turn-start-project" } as never),
322 }),
323 ),
324 ),
325 );
326
327 effectIt.effect("restores a missing worktree before a send only under the workspace lease", () =>
328 Effect.gen(function* () {
329 const worktreePath = `${managedWorktreesDir}/project/feature-restore`;
330 const { restore } = yield* ThreadWorktreeRestore;
331 yield* withWorkspaceLease(
332 worktreePath,
333 Effect.gen(function* () {
334 const restoring = yield* restore({
335 id: ThreadId.make("thread_send_restore_lease"),
336 projectId: ProjectId.make("project_send_restore_lease"),
337 worktreePath,
338 branch: "feature/restore",
339 }).pipe(Effect.forkChild({ startImmediately: true }));
340 expect(restoreLeaseExists).not.toHaveBeenCalled();
341 return restoring;
342 }),
343 ).pipe(Effect.flatMap(Fiber.join));
344 expect(restoreLeaseCreateWorktree).toHaveBeenCalledTimes(1);
345 }).pipe(Effect.provide(layerRestoreLeaseTest)),
346 );
347
348 effectIt.effect("restores into a leftover worktree folder that cannot itself be removed", () => {
349 const worktreePath = `${managedWorktreesDir}/project/feature-restore`;
350 // Windows refuses to remove a folder that is still some process's cwd.
351 const remove = vi.fn((target: string) =>
352 target === worktreePath
353 ? Effect.fail(
354 PlatformError.systemError({
355 _tag: "Busy",
356 module: "FileSystem",
357 method: "remove",
358 pathOrDescriptor: target,
359 }),
360 )
361 : Effect.void,
362 );
363 const createWorktree = vi.fn(() => Effect.succeed({} as never));
364 return Effect.gen(function* () {
365 const { restore } = yield* ThreadWorktreeRestore;
366 yield* restore({
367 id: ThreadId.make("thread_leftover_restore"),
368 projectId: ProjectId.make("project_leftover_restore"),
369 worktreePath,
370 branch: "feature/restore",
371 });
372 expect(remove).toHaveBeenCalledWith(`${worktreePath}/node_modules`, { recursive: true });
373 expect(createWorktree).toHaveBeenCalledTimes(1);
374 }).pipe(
375 Effect.provide(
376 layerThreadWorktreeRestore.pipe(
377 Layer.provide(
378 Layer.mergeAll(
379 Layer.succeed(FileSystem.FileSystem, {
380 exists: (target: string) => Effect.succeed(!target.endsWith("/.git")),
381 readDirectory: () => Effect.succeed(["node_modules"]),
382 stat: () => Effect.succeed({ type: "Directory" }),
383 realPath: (target: string) => Effect.succeed(normalize(target)),
384 remove,
385 } as never),
386 layerWorktreeLocation(),
387 Layer.mock(GitWorkflow.GitWorkflowService)({
388 pruneWorktrees: () => Effect.void,
389 createWorktree,
390 listLocalBranchNames: () => Effect.succeed(["feature/restore"]),
391 }),
392 Layer.mock(ProjectService.ProjectService)({
393 getById: () =>
394 Effect.succeedSome({ workspaceRoot: "/tmp/provider-turn-start-project" } as never),
395 }),
396 ),
397 ),
398 ),
399 ),
400 );
401 });
402
403 it.each([
404 { label: "directly in the scratch root", folder: "/tmp/t3-home/scratch/notes", recreated: true },
405 { label: "below another folder", folder: "/tmp/t3-home/scratch/notes/deep", recreated: false },
406 {
407 label: "in another project",
408 folder: "/tmp/elsewhere/notes",
409 workspaceRoot: "/tmp/elsewhere",
410 recreated: false,
411 },
412 ])("recreates a pruned No project folder only $label", async (input) => {
413 const makeDirectory = vi.fn(() => Effect.void);
414 await Effect.gen(function* () {
415 const { restore } = yield* ThreadWorktreeRestore;
416 yield* restore({
417 id: ThreadId.make("thread_scratch_restore"),
418 projectId: ProjectId.make("project_scratch_restore"),
419 worktreePath: input.folder,
420 branch: null,
421 });
422 }).pipe(
423 Effect.provide(
424 layerThreadWorktreeRestore.pipe(
425 Layer.provide(
426 Layer.mergeAll(
427 Layer.succeed(FileSystem.FileSystem, {
428 exists: () => Effect.succeed(false),
429 realPath: (target: string) => Effect.succeed(target),
430 makeDirectory,
431 } as never),
432 Path.layer,
433 Layer.mock(ServerConfig.ServerConfig)({
434 baseDir: "/tmp/t3-home",
435 worktreesDir: managedWorktreesDir,
436 } as ServerConfig.ServerConfig["Service"]),
437 ServerSettings.layerTest(),
438 Layer.mock(GitWorkflow.GitWorkflowService)({}),
439 Layer.mock(ProjectService.ProjectService)({
440 getById: () =>
441 Effect.succeedSome({
442 workspaceRoot: input.workspaceRoot ?? "/tmp/t3-home/scratch",
443 } as never),
444 }),
445 ),
446 ),
447 ),
448 ),
449 Effect.runPromise,
450 );
451 expect(makeDirectory).toHaveBeenCalledTimes(input.recreated ? 1 : 0);
452 });
453
454 function makeLocalCommandHarness(input: {
455 readonly text: string;
456 readonly manualContinuation?: boolean;
457 readonly nativeResumeUnsupported?: boolean;
458 readonly steeringRestart?: boolean;
459 readonly nativeGoals?: boolean;
460 readonly canForkWhileSourceRunning?: boolean;
461 readonly exposesNativeTurnId?: boolean;
462 readonly goalDriver?: "opencode" | "antigravity";
463 readonly worktreePath?: string;
464 readonly worktreesDir?: string;
465 readonly workspaceRoot?: string;
466 readonly gitWorkflow?: GitWorkflow.GitWorkflowService["Service"];
467 readonly fileSystem?: FileSystem.FileSystem;
468 readonly previousNativeSession?: boolean;
469 readonly previousMessages?: ReadonlyArray<string>;
470 readonly createdBy?: "agent" | "system";
471 readonly scheduledTaskId?: ScheduledTaskId;
472 readonly logoutFailure?: string;
473 readonly openFailure?: unknown;
474 readonly ensureThreadFailure?: unknown;
475 readonly historyReadFailureAfterFallback?: unknown;
476 readonly resumeFailure?: ProviderAdapter.ProviderAdapterV2Error;
477 readonly interruptOpen?: boolean;
478 readonly interruptRunBeforeOpenFailure?: boolean;
479 readonly writeFailure?: unknown;
480 readonly failReadsAfterRunning?: boolean;
481 readonly missingProviderSession?: boolean;
482 readonly recoveryReadFailure?: string;
483 readonly previousTurn?: {
484 readonly runStatus: "failed" | "completed" | "interrupted";
485 readonly turnStatus: "running" | "completed" | "interrupted";
486 readonly differentThread?: boolean;
487 readonly liveSession?: boolean;
488 readonly interruptFailure?: boolean;
489 readonly interruptCurrentRun?: boolean;
490 /** Puts the running turn on an attempt a steering restart superseded. */
491 readonly supersededTurn?: boolean;
492 };
493 /**
494 * A side chat whose context targets this run, an earlier run that failed or was cancelled before
495 * the provider, or an earlier run that delivered it. `nativeForkFrom` plans a native fork of the
496 * parent. `switchedProvider` puts the earlier run and its context on another provider thread.
497 */
498 readonly sideChat?: {
499 readonly contextRun: "current" | "failed" | "cancelled" | "delivered";
500 readonly unsentResume?: boolean;
501 readonly repeatedResume?: boolean;
502 readonly acceptedResume?: boolean;
503 readonly restartedResume?: boolean;
504 readonly nativeForkFrom?: "idle" | "busy";
505 readonly sourceBoundary?: "native" | "synthetic";
506 readonly sourceRolledBack?: boolean;
507 readonly switchedProvider?: boolean;
508 readonly threadRuntimeMode?: OrchestrationV2ThreadProjection["thread"]["runtimeMode"];
509 };
510 }) {
511 const settledRuns: Array<string> = [];
512 const now = DateTime.makeUnsafe("2026-09-04T12:00:00Z");
513 const threadId = ThreadId.make("thread-native-account-command");
514 const runId = RunId.make("run-native-account-command");
515 const rootNodeId = NodeId.make("root-native-account-command");
516 const attemptId = RunAttemptId.make("attempt-native-account-command");
517 const providerThreadId = ProviderThreadId.make("new-provider-thread");
518 const providerSessionId = ProviderSessionId.make("new-provider-session");
519 const oldProviderThreadId = ProviderThreadId.make("existing-native-provider-thread");
520 const oldInstanceId = ProviderInstanceId.make("antigravity-personal");
521 const newInstanceId = ProviderInstanceId.make("codex-personal");
522 const checkpointScopeId = CheckpointScopeId.make("scope-native-account-command");
523 const messageId = MessageId.make("message-native-account-command");
524 const run: OrchestrationV2ThreadProjection["runs"][number] = {
525 ...(input.manualContinuation ? { manualContinuationOfRunId: RunId.make("stopped-run") } : {}),
526 id: runId,
527 threadId,
528 ordinal: 2,
529 providerInstanceId: newInstanceId,
530 modelSelection: { instanceId: newInstanceId, model: "gpt-5.4" },
531 providerThreadId,
532 userMessageId: messageId,
533 rootNodeId,
534 activeAttemptId: attemptId,
535 status: "starting",
536 requestedAt: now,
537 startedAt: null,
538 completedAt: null,
539 checkpointId: null,
540 contextHandoffId: null,
541 };
542 const providerThread: OrchestrationV2ThreadProjection["providerThreads"][number] = {
543 id: providerThreadId,
544 driver: ProviderDriverKind.make(
545 input.nativeGoals === undefined ? "codex" : (input.goalDriver ?? "opencode"),
546 ),
547 providerInstanceId: newInstanceId,
548 providerSessionId: input.missingProviderSession === true ? null : providerSessionId,
549 appThreadId: threadId,
550 ownerNodeId: null,
551 nativeThreadRef: null,
552 nativeConversationHeadRef: null,
553 status: "not_loaded",
554 firstRunOrdinal: 2,
555 lastRunOrdinal: 2,
556 handoffIds: [],
557 forkedFrom: null,
558 createdAt: now,
559 updatedAt: now,
560 };
561 const message: OrchestrationV2ThreadProjection["messages"][number] = {
562 id: messageId,
563 threadId,
564 runId,
565 nodeId: rootNodeId,
566 role: "user",
567 text: input.text,
568 attachments: [],
569 streaming: false,
570 createdBy: input.createdBy ?? "user",
571 creationSource: input.createdBy === undefined ? "web" : "mcp",
572 ...(input.scheduledTaskId === undefined ? {} : { scheduledTaskId: input.scheduledTaskId }),
573 createdAt: now,
574 updatedAt: now,
575 };
576 let projection: OrchestrationV2ThreadProjection = {
577 thread: {
578 id: threadId,
579 activeProviderThreadId: providerThreadId,
580 projectId: ProjectId.make("project-native-account-command"),
581 branch: input.worktreePath === undefined ? null : "feature/restore",
582 worktreePath: input.worktreePath ?? null,
583 lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId },
584 } as OrchestrationV2ThreadProjection["thread"],
585 runs: [
586 ...(input.previousNativeSession
587 ? [
588 {
589 ...run,
590 id: RunId.make("previous-native-run"),
591 ordinal: 1,
592 status: "completed" as const,
593 providerInstanceId: oldInstanceId,
594 providerThreadId: oldProviderThreadId,
595 },
596 ]
597 : []),
598 run,
599 ],
600 attempts: [
601 {
602 id: attemptId,
603 runId,
604 rootNodeId,
605 attemptOrdinal: 1,
606 providerInstanceId: newInstanceId,
607 providerThreadId,
608 providerTurnId: null,
609 reason: input.steeringRestart ? "steering_restart" : "initial",
610 status: "pending",
611 startedAt: null,
612 completedAt: null,
613 },
614 ],
615 nodes: [
616 {
617 id: rootNodeId,
618 threadId,
619 runId,
620 parentNodeId: null,
621 rootNodeId,
622 kind: "root_turn",
623 status: "pending",
624 countsForRun: true,
625 providerThreadId,
626 providerTurnId: null,
627 nativeItemRef: null,
628 runtimeRequestId: null,
629 checkpointScopeId,
630 startedAt: null,
631 completedAt: null,
632 },
633 ],
634 providerThreads: [
635 ...(input.previousNativeSession
636 ? [
637 {
638 ...providerThread,
639 id: oldProviderThreadId,
640 providerInstanceId: oldInstanceId,
641 driver: ProviderDriverKind.make("antigravity"),
642 lastRunOrdinal: 1,
643 nativeThreadRef: {
644 driver: ProviderDriverKind.make("antigravity"),
645 nativeId: "existing-session",
646 strength: "strong" as const,
647 },
648 },
649 ]
650 : []),
651 providerThread,
652 ],
653 messages: [
654 ...(input.previousMessages ?? []).map((text, index) => ({
655 ...message,
656 id: MessageId.make(`previous-message-${index}`),
657 text,
658 })),
659 message,
660 ],
661 checkpointScopes: [
662 {
663 id: checkpointScopeId,
664 threadId,
665 runId,
666 nodeId: rootNodeId,
667 parentScopeId: null,
668 providerThreadId,
669 kind: "root_run",
670 ordinalWithinParent: 0,
671 advancesAppRunCount: true,
672 cwd: "/tmp/native-account-command",
673 createdAt: now,
674 },
675 ],
676 providerSessions: [],
677 providerTurns: [],
678 contextHandoffs: [],
679 contextTransfers: [],
680 turnItems: [],
681 visibleTurnItems: [],
682 runtimeRequests: [],
683 subagents: [],
684 plans: [],
685 checkpoints: [],
686 updatedAt: now,
687 };
688 const parentThreadId = ThreadId.make("side-chat-parent");
689 const parentRunId = RunId.make("side-chat-parent-run");
690 const parentProviderThread = {
691 ...providerThread,
692 id: ProviderThreadId.make("side-chat-parent-provider-thread"),
693 appThreadId: parentThreadId,
694 nativeThreadRef: {
695 driver: providerThread.driver,
696 nativeId: "side-chat-parent-native",
697 strength: "strong" as const,
698 },
699 };
700 const parentProviderTurn = (ordinal: number, attemptRunId: RunId) => ({
701 attempt: {
702 ...projection.attempts[0]!,
703 id: RunAttemptId.make(`side-chat-parent-attempt-${ordinal}`),
704 runId: attemptRunId,
705 providerThreadId: parentProviderThread.id,
706 status: "completed" as const,
707 },
708 turn: {
709 id: ProviderTurnId.make(`side-chat-parent-turn-${ordinal}`),
710 providerThreadId: parentProviderThread.id,
711 nodeId: rootNodeId,
712 runAttemptId: RunAttemptId.make(`side-chat-parent-attempt-${ordinal}`),
713 nativeTurnRef:
714 ordinal === 1 && input.sideChat?.sourceBoundary === "synthetic"
715 ? null
716 : {
717 driver: providerThread.driver,
718 nativeId: `side-chat-parent-native-turn-${ordinal}`,
719 strength: "strong" as const,
720 },
721 ordinal,
722 status: "completed" as const,
723 startedAt: now,
724 completedAt: now,
725 },
726 });
727 const parentRolledBackRunId = RunId.make("side-chat-parent-rolled-back-run");
728 const parentTurns = [
729 parentProviderTurn(1, parentRunId),
730 parentProviderTurn(2, parentRolledBackRunId),
731 ];
732 const parentRecords = {
733 ...projection,
734 thread: { ...projection.thread, id: parentThreadId },
735 runs: [
736 {
737 ...run,
738 id: parentRunId,
739 threadId: parentThreadId,
740 ordinal: 1,
741 providerThreadId: parentProviderThread.id,
742 activeAttemptId: parentTurns[0]!.attempt.id,
743 status:
744 input.sideChat?.sourceRolledBack === true
745 ? ("rolled_back" as const)
746 : ("completed" as const),
747 completedAt: now,
748 },
749 {
750 ...run,
751 id: parentRolledBackRunId,
752 threadId: parentThreadId,
753 ordinal: 2,
754 providerThreadId: parentProviderThread.id,
755 activeAttemptId: parentTurns[1]!.attempt.id,
756 status: "rolled_back" as const,
757 completedAt: now,
758 },
759 ...(input.sideChat?.nativeForkFrom === "busy"
760 ? [
761 {
762 ...run,
763 id: RunId.make("side-chat-parent-busy-run"),
764 threadId: parentThreadId,
765 ordinal: 3,
766 providerThreadId: parentProviderThread.id,
767 status: "running" as const,
768 },
769 ]
770 : []),
771 ],
772 providerThreads: [parentProviderThread],
773 attempts: parentTurns.map(({ attempt }) => attempt),
774 providerTurns: parentTurns.map(({ turn }) => turn),
775 turnItems: [],
776 };
777 if (input.sideChat !== undefined) {
778 const earlierRunId = RunId.make("side-chat-earlier-run");
779 projection = {
780 ...projection,
781 thread: {
782 ...projection.thread,
783 runtimeMode: input.sideChat.threadRuntimeMode ?? "full-access",
784 lineage: {
785 parentThreadId,
786 relationshipToParent: "fork",
787 rootThreadId: parentThreadId,
788 sideChat: true,
789 },
790 },
791 runs:
792 input.sideChat.contextRun === "current"
793 ? projection.runs
794 : [
795 {
796 ...run,
797 id: earlierRunId,
798 ordinal: 1,
799 userMessageId: MessageId.make("side-chat-earlier-message"),
800 activeAttemptId: RunAttemptId.make("side-chat-earlier-attempt"),
801 ...(input.sideChat.switchedProvider
802 ? { providerThreadId: oldProviderThreadId }
803 : {}),
804 status:
805 input.sideChat.contextRun === "delivered"
806 ? "completed"
807 : input.sideChat.contextRun,
808 completedAt: now,
809 },
810 run,
811 ],
812 contextTransfers:
813 input.sideChat.nativeForkFrom === undefined
814 ? []
815 : [
816 {
817 id: ContextTransferId.make("side-chat-fork"),
818 type: "fork",
819 sourceThreadId: parentThreadId,
820 targetThreadId: threadId,
821 sourcePoint: { threadId: parentThreadId, runId: parentRunId },
822 basePoint: null,
823 sourceProviderInstanceId: newInstanceId,
824 targetProviderInstanceId: newInstanceId,
825 targetRunId: runId,
826 status: "pending",
827 resolution: null,
828 createdBy: "user",
829 error: null,
830 createdAt: now,
831 updatedAt: now,
832 consumedAt: null,
833 },
834 ],
835 contextHandoffs: [
836 {
837 id: ContextHandoffId.make("side-chat-context"),
838 transferId: ContextTransferId.make("side-chat-fork"),
839 threadId,
840 targetRunId: input.sideChat.contextRun === "current" ? runId : earlierRunId,
841 fromProviderThreadIds: [],
842 toProviderThreadId: input.sideChat.switchedProvider
843 ? oldProviderThreadId
844 : providerThreadId,
845 coveredRunOrdinals: { from: 1, to: 1 },
846 strategy: "manual_context",
847 status: "ready",
848 summaryMessageId: null,
849 summaryText: "SIDE_CHAT_CONTEXT",
850 createdByProviderInstanceId: null,
851 createdAt: now,
852 updatedAt: now,
853 ...(input.sideChat.contextRun === "delivered"
854 ? {
855 delivery: {
856 nativeThreadId: "side-chat-native",
857 status: "inline" as const,
858 itemIds: [],
859 },
860 }
861 : {}),
862 },
863 ],
864 };
865 }
866 if (input.sideChat?.unsentResume) {
867 const originalMessageId = MessageId.make("original-unsent-message");
868 projection = {
869 ...projection,
870 runs: [
871 {
872 ...run,
873 id: RunId.make("stopped-run"),
874 manualContinuationOfRunId: undefined,
875 userMessageId: originalMessageId,
876 status: "interrupted",
877 ordinal: 1,
878 },
879 ...projection.runs,
880 ],
881 messages: [
882 { ...message, id: originalMessageId, text: "Original request" },
883 ...projection.messages,
884 ],
885 };
886 }
887 if (input.sideChat?.repeatedResume) {
888 const retry = RunId.make("stopped-resume");
889 projection = {
890 ...projection,
891 runs: projection.runs
892 .map((r) => (r.id === runId ? { ...r, manualContinuationOfRunId: retry } : r))
893 .concat({
894 ...run,
895 id: retry,
896 manualContinuationOfRunId: RunId.make("stopped-run"),
897 status: "interrupted",
898 }),
899 };
900 }
901 if (input.sideChat?.acceptedResume) {
902 const acceptedId = RunAttemptId.make("accepted-original");
903 projection = {
904 ...projection,
905 attempts: [
906 ...projection.attempts,
907 {
908 ...projection.attempts[0]!,
909 id: acceptedId,
910 runId: RunId.make("stopped-run"),
911 providerThreadId: oldProviderThreadId,
912 status: "interrupted",
913 },
914 ...(input.sideChat.restartedResume
915 ? [
916 {
917 ...projection.attempts[0]!,
918 id: RunAttemptId.make("unaccepted-replacement"),
919 runId: RunId.make("stopped-run"),
920 attemptOrdinal: 2,
921 reason: "steering_restart" as const,
922 status: "interrupted" as const,
923 },
924 ]
925 : []),
926 ],
927 providerTurns: [
928 ...projection.providerTurns,
929 {
930 id: ProviderTurnId.make("accepted-original-turn"),
931 runAttemptId: acceptedId,
932 providerThreadId: oldProviderThreadId,
933 nodeId: rootNodeId,
934 ordinal: 1,
935 nativeTurnRef: null,
936 status: "interrupted",
937 startedAt: now,
938 completedAt: now,
939 },
940 ],
941 };
942 }
943 if ("historyReadFailureAfterFallback" in input || "resumeFailure" in input) {
944 const nativeThreadRef = {
945 driver: providerThread.driver,
946 nativeId: "native-resume-thread",
947 strength: "strong" as const,
948 };
949 projection = {
950 ...projection,
951 providerThreads: projection.providerThreads.map((candidate) =>
952 candidate.id === providerThreadId ? { ...candidate, nativeThreadRef } : candidate,
953 ),
954 };
955 }
956 const previousTurnId = ProviderTurnId.make("previous-provider-turn");
957 if (input.previousTurn !== undefined) {
958 const previousRunId = RunId.make("previous-failed-run");
959 const previousAttemptId = RunAttemptId.make("previous-failed-attempt");
960 const steeredAttemptId = RunAttemptId.make("previous-steered-attempt");
961 const previousProviderThreadId = input.previousTurn.differentThread
962 ? oldProviderThreadId
963 : providerThreadId;
964 projection = {
965 ...projection,
966 attempts: [
967 {
968 id: previousAttemptId,
969 runId: previousRunId,
970 attemptOrdinal: 1,
971 rootNodeId: NodeId.make("previous-root"),
972 providerInstanceId: newInstanceId,
973 providerThreadId: previousProviderThreadId,
974 providerTurnId: null,
975 reason: "initial",
976 status: input.previousTurn.supersededTurn === true ? "superseded" : "interrupted",
977 startedAt: now,
978 completedAt: now,
979 },
980 ...(input.previousTurn.supersededTurn === true
981 ? [
982 {
983 id: steeredAttemptId,
984 runId: previousRunId,
985 attemptOrdinal: 2,
986 rootNodeId: NodeId.make("previous-steered-root"),
987 providerInstanceId: newInstanceId,
988 providerThreadId: previousProviderThreadId,
989 providerTurnId: null,
990 reason: "steering_restart" as const,
991 status: "interrupted" as const,
992 startedAt: null,
993 completedAt: now,
994 },
995 ]
996 : []),
997 ...projection.attempts,
998 ],
999 runs: [
1000 {
1001 ...run,
1002 id: previousRunId,
1003 ordinal: 1,
1004 status: input.previousTurn.runStatus,
1005 rootNodeId:
1006 input.previousTurn.supersededTurn === true
1007 ? NodeId.make("previous-steered-root")
1008 : NodeId.make("previous-root"),
1009 activeAttemptId:
1010 input.previousTurn.supersededTurn === true ? steeredAttemptId : previousAttemptId,
1011 providerThreadId: previousProviderThreadId,
1012 completedAt: now,
1013 },
1014 run,
1015 ],
1016 providerThreads: [
1017 {
1018 ...providerThread,
1019 nativeThreadRef: {
1020 driver: providerThread.driver,
1021 nativeId: "existing-native-conversation",
1022 strength: "strong",
1023 },
1024 },
1025 ],
1026 providerTurns: [
1027 // The steered attempt reached the provider once; only the earlier attempt's
1028 // turn is still running when the run was interrupted.
1029 ...(input.previousTurn.supersededTurn === true
1030 ? [
1031 {
1032 id: ProviderTurnId.make("previous-steered-turn"),
1033 providerThreadId: previousProviderThreadId,
1034 nodeId: NodeId.make("previous-steered-root"),
1035 runAttemptId: steeredAttemptId,
1036 ordinal: 0,
1037 nativeTurnRef: null,
1038 status: "interrupted" as const,
1039 startedAt: now,
1040 completedAt: now,
1041 },
1042 ]
1043 : []),
1044 {
1045 id: previousTurnId,
1046 providerThreadId: previousProviderThreadId,
1047 nodeId: NodeId.make("previous-root"),
1048 runAttemptId: previousAttemptId,
1049 ordinal: 1,
1050 nativeTurnRef: null,
1051 status: input.previousTurn.turnStatus,
1052 startedAt: now,
1053 completedAt: input.previousTurn.turnStatus === "running" ? null : now,
1054 },
1055 ],
1056 };
1057 }
1058 let providerBusy =
1059 (input.previousTurn?.runStatus === "failed" ||
1060 input.previousTurn?.runStatus === "interrupted") &&
1061 input.previousTurn.turnStatus === "running" &&
1062 !input.previousTurn.differentThread &&
1063 input.previousTurn.liveSession !== false;
1064 const interruptTurn = vi.fn((request: ProviderAdapter.ProviderAdapterV2InterruptInput) =>
1065 Effect.gen(function* () {
1066 expect(request.providerTurnId).toBe(previousTurnId);
1067 expect(request.providerThread.id).toBe(providerThreadId);
1068 if (input.previousTurn?.interruptFailure) {
1069 return yield* new ProviderAdapter.ProviderAdapterInterruptError({
1070 driver: providerThread.driver,
1071 providerThreadId,
1072 providerTurnId: request.providerTurnId,
1073 cause: "provider interrupt failed",
1074 });
1075 }
1076 providerBusy = false;
1077 if (input.previousTurn?.interruptCurrentRun) interruptRun();
1078 }),
1079 );
1080 const events: Array<OrchestrationV2DomainEvent> = [];
1081 const interruptRun = () => {
1082 projection = {
1083 ...projection,
1084 runs: projection.runs.map((candidate) =>
1085 candidate.id === runId
1086 ? { ...candidate, status: "interrupted", completedAt: now }
1087 : candidate,
1088 ),
1089 };
1090 };
1091 const ensureThread = vi.fn(() =>
1092 Effect.sync(() => {
1093 if (input.interruptRunBeforeOpenFailure === true) interruptRun();
1094 }).pipe(
1095 Effect.andThen(
1096 Effect.fail(
1097 new ProviderAdapter.ProviderAdapterEventStreamError({
1098 driver: providerThread.driver,
1099 providerSessionId,
1100 cause: input.ensureThreadFailure,
1101 }),
1102 ),
1103 ),
1104 ),
1105 );
1106 const fallbackEnsureThread = vi.fn(() => Effect.succeed(providerThread));
1107 const providerSession = {
1108 id: providerSessionId,
1109 driver: providerThread.driver,
1110 providerInstanceId: newInstanceId,
1111 status: "ready",
1112 cwd: "/tmp/native-account-command",
1113 model: null,
1114 capabilities: {
1115 ...CodexProviderCapabilitiesV2,
1116 threads: {
1117 ...CodexProviderCapabilitiesV2.threads,
1118 canForkWhileSourceRunning:
1119 input.canForkWhileSourceRunning ??
1120 CodexProviderCapabilitiesV2.threads.canForkWhileSourceRunning,
1121 },
1122 turns: {
1123 ...CodexProviderCapabilitiesV2.turns,
1124 exposesNativeTurnId:
1125 input.exposesNativeTurnId ?? CodexProviderCapabilitiesV2.turns.exposesNativeTurnId,
1126 ...(input.nativeGoals === undefined ? {} : { supportsNativeGoals: input.nativeGoals }),
1127 },
1128 },
1129 createdAt: now,
1130 updatedAt: now,
1131 lastError: null,
1132 };
1133 const resumeFallbackSession = {
1134 driver: providerThread.driver,
1135 providerSession,
1136 resumeThread: () =>
1137 Effect.fail(
1138 input.resumeFailure ??
1139 new ProviderAdapter.ProviderAdapterEventStreamError({
1140 driver: providerThread.driver,
1141 providerSessionId,
1142 cause: "native thread is gone",
1143 }),
1144 ),
1145 ensureThread: fallbackEnsureThread,
1146 };
1147 const startTurn = vi.fn((_turn: ProviderAdapter.ProviderAdapterV2TurnInput) => Effect.void);
1148 const sideChatThread = {
1149 ...providerThread,
1150 nativeThreadRef: {
1151 driver: providerThread.driver,
1152 nativeId: "side-chat-native",
1153 strength: "strong" as const,
1154 },
1155 };
1156 const forkThread = vi.fn<ProviderAdapter.ProviderAdapterV2SessionRuntime["forkThread"]>(() =>
1157 Effect.succeed(sideChatThread),
1158 );
1159 const sideChatSession = {
1160 driver: providerThread.driver,
1161 providerSession,
1162 ensureThread: () => Effect.succeed(sideChatThread),
1163 resumeThread: () => Effect.succeed(sideChatThread),
1164 forkThread,
1165 startTurn,
1166 resumeTurn: input.nativeResumeUnsupported
1167 ? undefined
1168 : (input: ProviderAdapter.ProviderAdapterV2ResumeTurnInput) =>
1169 startTurn(ProviderAdapter.resumeTurnInput(input)),
1170 };
1171 const open = vi.fn(() =>
1172 input.sideChat !== undefined
1173 ? Effect.succeed(sideChatSession as never)
1174 : input.interruptOpen === true
1175 ? Effect.interrupt
1176 : "historyReadFailureAfterFallback" in input || "resumeFailure" in input
1177 ? Effect.succeed(resumeFallbackSession as never)
1178 : "ensureThreadFailure" in input
1179 ? Effect.succeed({ driver: providerThread.driver, ensureThread } as never)
1180 : "openFailure" in input
1181 ? Effect.sync(() => {
1182 if (input.interruptRunBeforeOpenFailure === true) interruptRun();
1183 }).pipe(
1184 Effect.andThen(
1185 Effect.fail(
1186 new ProviderSessionManager.ProviderSessionOpenError({
1187 instanceId: newInstanceId,
1188 providerSessionId,
1189 cause: input.openFailure,
1190 }),
1191 ),
1192 ),
1193 )
1194 : input.failReadsAfterRunning === true || input.previousTurn !== undefined
1195 ? Effect.succeed({
1196 driver: providerThread.driver,
1197 providerSession,
1198 ensureThread: () => Effect.succeed(providerThread),
1199 resumeThread: () => Effect.succeed(projection.providerThreads[0]),
1200 } as never)
1201 : Effect.die("A local command must not open a native session."),
1202 );
1203 const startRootRun = vi.fn<
1204 (
1205 input: RunExecutionService.RunExecutionServiceV2StartRootRunInput,
1206 ) => Effect.Effect<void, RunExecutionService.RunExecutionServiceV2Error>
1207 >((runInput) =>
1208 input.sideChat !== undefined
1209 ? (runInput.resume
1210 ? runInput.session.resumeTurn!({
1211 ...runInput,
1212 messageId: runInput.message.messageId,
1213 } as never)
1214 : runInput.session.startTurn(runInput as never)
1215 ).pipe(Effect.orDie)
1216 : input.failReadsAfterRunning === true || input.previousTurn !== undefined
1217 ? Effect.suspend(() =>
1218 providerBusy ? Effect.die("The previous provider turn is still running.") : Effect.void,
1219 )
1220 : Effect.die("A local command must not start a native turn."),
1221 );
1222 const failReadIfRunning = Effect.suspend(() =>
1223 input.failReadsAfterRunning === true &&
1224 projection.runs.find((candidate) => candidate.id === runId)?.status === "running"
1225 ? Effect.fail(
1226 new ProjectionStore.ProjectionStoreReadError({ threadId, cause: "database unavailable" }),
1227 )
1228 : Effect.void,
1229 );
1230 const tryHandlePromptCommand = vi.fn(() =>
1231 input.logoutFailure === undefined
1232 ? Effect.succeed(true)
1233 : Effect.fail(
1234 new ProviderSetupError({
1235 instanceId: oldInstanceId,
1236 operation: "logout",
1237 detail: input.logoutFailure,
1238 }),
1239 ),
1240 );
1241 const writeIfRunCurrent = vi.fn<EventSink.EventSinkV2["Service"]["writeIfRunCurrent"]>(
1242 ({ events: incoming, activeAttemptId, expectedStatus }) =>
1243 "writeFailure" in input
1244 ? Effect.fail(
1245 new EventSink.EventSinkWriteError({
1246 eventCount: incoming.length,
1247 cause: input.writeFailure,
1248 }),
1249 )
1250 : Effect.sync(() => {
1251 const current = projection.runs.find((candidate) => candidate.id === runId);
1252 const committed =
1253 current !== undefined &&
1254 current.activeAttemptId === activeAttemptId &&
1255 current.status === expectedStatus;
1256 if (committed) {
1257 for (const event of incoming) {
1258 expect(isDomainEvent(event)).toBe(true);
1259 events.push(event);
1260 projection = ProjectionStore.applyToProjection(projection, event);
1261 }
1262 }
1263 return { committed, storedEvents: [] };
1264 }),
1265 );
1266 const resolveRuntimePolicy = vi.fn(
1267 ({ thread }: Parameters<RuntimePolicy.RuntimePolicyV2Shape["resolve"]>[0]) =>
1268 Effect.succeed({
1269 cwd: thread.worktreePath ?? "/tmp/native-account-command",
1270 runtimeMode: thread.runtimeMode,
1271 } as never),
1272 );
1273 const createWorktree = vi.fn<GitWorkflow.GitWorkflowService["Service"]["createWorktree"]>(() =>
1274 Effect.succeed({} as never),
1275 );
1276 const getThreadRecords = vi.fn<ProjectionStore.ProjectionStoreV2Shape["getThreadRecords"]>((id) =>
1277 Effect.succeed(id === parentThreadId ? parentRecords : projection),
1278 );
1279 const prepareProviderHandoff = vi.fn<
1280 ContextHandoffService.ContextHandoffServiceV2["Service"]["prepareProviderHandoff"]
1281 >((handoff) =>
1282 input.sideChat?.nativeForkFrom === undefined
1283 ? Effect.die("history read must fail first")
1284 : Effect.succeed({
1285 ...handoff,
1286 id: ContextHandoffId.make("side-chat-fork-context"),
1287 createdByProviderInstanceId: handoff.toProviderInstanceId,
1288 status: "ready",
1289 summaryMessageId: null,
1290 summaryText: "PARENT_HISTORY",
1291 updatedAt: handoff.createdAt,
1292 }),
1293 );
1294 const getTurnStartContext = vi.fn(() =>
1295 Effect.succeed({
1296 ...projection,
1297 hasConversation: projection.messages.some(
1298 (m) =>
1299 m.role === "user" &&
1300 (m.text.trim().toLowerCase() !== "/compact" || m.attachments.length > 0),
1301 ),
1302 }),
1303 );
1304 const getRuntimeRecoveryProjection = vi.fn<
1305 ProjectionStore.ProjectionStoreV2Shape["getRuntimeRecoveryProjection"]
1306 >(() =>
1307 input.recoveryReadFailure !== undefined
1308 ? Effect.fail(
1309 new ProjectionStore.ProjectionStoreReadError({
1310 threadId,
1311 cause: input.recoveryReadFailure,
1312 }),
1313 )
1314 : Effect.as(failReadIfRunning, {
1315 ...projection,
1316 hasConversation: projection.messages.some(
1317 (m) =>
1318 m.role === "user" &&
1319 (m.text.trim().toLowerCase() !== "/compact" || m.attachments.length > 0),
1320 ),
1321 }),
1322 );
1323 const getRun = vi.fn<ProjectionStore.ProjectionStoreV2Shape["getRun"]>((_threadId, id) =>
1324 Effect.as(
1325 failReadIfRunning,
1326 projection.runs.find((candidate) => candidate.id === id),
1327 ),
1328 );
1329 const layer = ProviderTurnStart.layer.pipe(
1330 Layer.provide(
1331 Layer.mergeAll(
1332 Layer.mock(ContextHandoffService.ContextHandoffServiceV2)({
1333 prepareProviderHandoff,
1334 }),
1335 Layer.mock(EventSink.EventSinkV2)({
1336 writeIfRunCurrent,
1337 write: ({ events: incoming }) =>
1338 Effect.sync(() => {
1339 for (const event of incoming) {
1340 events.push(event);
1341 projection = ProjectionStore.applyToProjection(projection, event);
1342 }
1343 return [];
1344 }),
1345 }),
1346 IdAllocator.layer,
1347 Layer.mock(TurnItemPositionStore.TurnItemPositionStoreV2)({}),
1348 layerWorktreeLocation(input.worktreesDir),
1349 input.fileSystem === undefined
1350 ? FileSystem.layerNoop({ exists: () => Effect.succeed(true) })
1351 : Layer.succeed(FileSystem.FileSystem, input.fileSystem),
1352 input.gitWorkflow === undefined
1353 ? Layer.mock(GitWorkflow.GitWorkflowService)({
1354 pruneWorktrees: () => Effect.void,
1355 listLocalBranchNames: () => Effect.succeed(["feature/restore"]),
1356 createWorktree,
1357 })
1358 : Layer.succeed(GitWorkflow.GitWorkflowService, input.gitWorkflow),
1359 Layer.mock(ProjectService.ProjectService)({
1360 getById: () =>
1361 Effect.succeedSome({
1362 workspaceRoot: input.workspaceRoot ?? "/tmp/native-account-project",
1363 } as never),
1364 }),
1365 Layer.mock(ProjectionStore.ProjectionStoreV2)({
1366 getThreadRecords,
1367 getTurnStartContext,
1368 getRuntimeRecoveryProjection,
1369 getRun,
1370 getTurnStartHistory: () =>
1371 input.sideChat !== undefined
1372 ? Effect.succeed([])
1373 : Effect.fail(
1374 new ProjectionStore.ProjectionStoreReadError({
1375 threadId,
1376 cause: input.historyReadFailureAfterFallback,
1377 }),
1378 ),
1379 }),
1380 Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({
1381 open,
1382 get: () =>
1383 Effect.succeed(
1384 input.previousTurn?.liveSession === false
1385 ? Option.none()
1386 : Option.some({
1387 driver: providerThread.driver,
1388 providerSessionId,
1389 interruptTurn,
1390 } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime),
1391 ),
1392 }),
1393 Layer.mock(ProviderAuthService.ProviderAuthService)({ tryHandlePromptCommand }),
1394 Layer.mock(RunExecutionService.RunExecutionServiceV2)({
1395 startRootRun,
1396 forkBaseline: () => Effect.succeed(Effect.void),
1397 }),
1398 Layer.succeed(ProviderUsageAnalytics.ProviderTurnAnalytics, {
1399 runStarted: () => Effect.void,
1400 turnCompleted: () => Effect.void,
1401 runSettled: (report) => Effect.sync(() => void settledRuns.push(report.status)),
1402 }),
1403 Layer.mock(RuntimePolicy.RuntimePolicyV2)({
1404 resolve: resolveRuntimePolicy,
1405 }),
1406 ),
1407 ),
1408 );
1409 return {
1410 layer,
1411 interruptRun,
1412 settledRuns,
1413 getThreadRecords,
1414 getTurnStartContext,
1415 getRuntimeRecoveryProjection,
1416 getRun,
1417 setProjection: (next: OrchestrationV2ThreadProjection) => {
1418 projection = next;
1419 },
1420 resolveRuntimePolicy,
1421 createWorktree,
1422 open,
1423 fallbackEnsureThread,
1424 writeIfRunCurrent,
1425 startRootRun,
1426 startTurn,
1427 forkThread,
1428 prepareProviderHandoff,
1429 interruptTurn,
1430 tryHandlePromptCommand,
1431 events,
1432 oldInstanceId,
1433 newInstanceId,
1434 attemptId,
1435 projection: () => projection,
1436 start: Effect.gen(function* () {
1437 yield* (yield* ProviderTurnStart.ProviderTurnStartServiceV2).start({ threadId, runId });
1438 }).pipe(Effect.provide(layer)),
1439 startWithRetry: Effect.gen(function* () {
1440 yield* (yield* ProviderTurnStart.ProviderTurnStartServiceV2).start({
1441 threadId,
1442 runId,
1443 willRetry: true,
1444 });
1445 }).pipe(Effect.provide(layer)),
1446 };
1447 }
1448
1449 function layerStartEffectWorker(
1450 harness: ReturnType<typeof makeLocalCommandHarness>,
1451 control: Partial<ProviderTurnControlService.ProviderTurnControlServiceV2["Service"]> = {},
1452 ) {
1453 const executor = EffectWorker.layerExecutor.pipe(
1454 Layer.provide(
1455 Layer.mergeAll(
1456 harness.layer,
1457 Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({}),
1458 Layer.mock(ProviderTurnControlService.ProviderTurnControlServiceV2)(control),
1459 Layer.mock(RunFinalizationService.RunFinalizationService)({}),
1460 Layer.mock(CheckpointRollbackService.CheckpointRollbackServiceV2)({}),
1461 Layer.mock(RuntimeRequestService.RuntimeRequestServiceV2)({}),
1462 Layer.mock(ThreadTitleRegenerationService.ThreadTitleRegenerationService)({}),
1463 Layer.mock(ThreadManagementService.ThreadManagementService)({}),
1464 Layer.mock(ProjectionStore.ProjectionStoreV2)({}),
1465 ServerSettings.layerTest(),
1466 ),
1467 ),
1468 );
1469 return EffectWorker.layerWithOptions({ maxAttempts: 2 }).pipe(
1470 Layer.provideMerge(
1471 Layer.merge(executor, EffectOutbox.layer.pipe(Layer.provide(SqlitePersistence.layerMemory))),
1472 ),
1473 );
1474 }
1475
1476 const enqueueStart = (
1477 projection: OrchestrationV2ThreadProjection,
1478 request?: EffectOutbox.OrchestrationEffectRequestV2,
1479 ) =>
1480 Effect.gen(function* () {
1481 const outbox = yield* EffectOutbox.EffectOutboxV2;
1482 const run = projection.runs.at(-1)!;
1483 yield* outbox.enqueue([
1484 {
1485 id: "effect:failed-start",
1486 commandId: CommandId.make("command:failed-start"),
1487 threadId: run.threadId,
1488 request: request ?? { type: "provider-turn.start", runId: run.id },
1489 },
1490 ]);
1491 return { storedEffect: outbox.get("effect:failed-start").pipe(Effect.map(Option.getOrThrow)) };
1492 });
1493
1494 effectIt.effect(
1495 "settles an exhausted start effect without claiming an external turn stopped",
1496 () => {
1497 const harness = makeLocalCommandHarness({ text: "Continue", missingProviderSession: true });
1498 const projection = harness.projection();
1499 const previousAttempt = {
1500 ...projection.attempts[0]!,
1501 id: RunAttemptId.make("superseded-attempt"),
1502 rootNodeId: NodeId.make("superseded-root"),
1503 status: "superseded" as const,
1504 };
1505 const externalTurn = {
1506 id: ProviderTurnId.make("still-running-provider-turn"),
1507 providerThreadId: projection.providerThreads[0]!.id,
1508 nodeId: previousAttempt.rootNodeId,
1509 runAttemptId: previousAttempt.id,
1510 nativeTurnRef: null,
1511 ordinal: 1,
1512 status: "running" as const,
1513 startedAt: projection.updatedAt,
1514 completedAt: null,
1515 };
1516 harness.setProjection({
1517 ...projection,
1518 attempts: [
1519 previousAttempt,
1520 { ...projection.attempts[0]!, attemptOrdinal: 2, reason: "steering_restart" },
1521 ],
1522 providerTurns: [externalTurn],
1523 });
1524
1525 return Effect.gen(function* () {
1526 const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
1527 const { storedEffect } = yield* enqueueStart(harness.projection());
1528 expect(yield* worker.drain()).toBe(1);
1529 expect(yield* storedEffect).toMatchObject({ status: "pending", attemptCount: 1 });
1530 expect(harness.projection().runs.at(-1)?.status).toBe("starting");
1531 expect(harness.events).toEqual([]);
1532
1533 yield* TestClock.adjust("100 millis");
1534 expect(yield* worker.drain()).toBe(1);
1535 expect(yield* storedEffect).toMatchObject({ status: "succeeded", attemptCount: 2 });
1536 expect(harness.projection().runs.at(-1)).toMatchObject({ status: "failed", startedAt: null });
1537 expect(harness.projection().attempts.at(-1)).toMatchObject({
1538 status: "failed",
1539 startedAt: null,
1540 });
1541 expect(harness.projection().nodes[0]).toMatchObject({ status: "failed", startedAt: null });
1542 expect(harness.projection().turnItems).toMatchObject([
1543 {
1544 type: "error",
1545 status: "failed",
1546 title: "Turn failed to start",
1547 failure: {
1548 class: "unknown",
1549 message:
1550 "T3 Code could not start this turn. Send the message again; if it keeps failing, check the server logs.",
1551 },
1552 },
1553 ]);
1554 expect(harness.open).not.toHaveBeenCalled();
1555 expect(harness.startRootRun).not.toHaveBeenCalled();
1556 expect(harness.projection().providerTurns).toEqual([externalTurn]);
1557 }).pipe(Effect.provide(layerStartEffectWorker(harness)));
1558 },
1559 );
1560
1561 effectIt.effect("retries a failed settlement past the start budget until it is persisted", () => {
1562 const harness = makeLocalCommandHarness({ text: "Continue", missingProviderSession: true });
1563 harness.writeIfRunCurrent.mockImplementationOnce(({ events }) =>
1564 Effect.fail(
1565 new EventSink.EventSinkWriteError({
1566 eventCount: events.length,
1567 cause: "storage unavailable",
1568 }),
1569 ),
1570 );
1571
1572 return Effect.gen(function* () {
1573 const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
1574 const { storedEffect } = yield* enqueueStart(harness.projection());
1575 yield* worker.drain();
1576 yield* TestClock.adjust("100 millis");
1577 expect(yield* worker.drain()).toBe(1);
1578 expect(yield* storedEffect).toMatchObject({ status: "pending", attemptCount: 2 });
1579 expect(harness.projection().runs.at(-1)?.status).toBe("starting");
1580 expect(harness.events).toEqual([]);
1581
1582 yield* TestClock.adjust("200 millis");
1583 expect(yield* worker.drain()).toBe(1);
1584 expect(yield* storedEffect).toMatchObject({ status: "succeeded", attemptCount: 3 });
1585 expect(harness.projection().runs.at(-1)?.status).toBe("failed");
1586 expect(harness.projection().turnItems).toHaveLength(1);
1587 }).pipe(Effect.provide(layerStartEffectWorker(harness)));
1588 });
1589
1590 effectIt.effect.each(["interrupt", "start"] as const)(
1591 "keeps a failed restart %s separate from starting-run settlement",
1592 (stage) => {
1593 const harness = makeLocalCommandHarness({ text: "Continue", missingProviderSession: true });
1594 const projection = harness.projection();
1595 const run = projection.runs.at(-1)!;
1596 const providerTurnId = ProviderTurnId.make("restart-interrupted-turn");
1597 return Effect.gen(function* () {
1598 const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
1599 const { storedEffect } = yield* enqueueStart(projection, {
1600 type: "provider-turn.restart",
1601 runId: run.id,
1602 providerSessionId: ProviderSessionId.make("restart-interrupted-session"),
1603 providerThreadId: projection.providerThreads[0]!.id,
1604 providerTurnId,
1605 interruptedAttemptId: RunAttemptId.make("restart-interrupted-attempt"),
1606 });
1607 yield* worker.drain();
1608 yield* TestClock.adjust("100 millis");
1609 expect(yield* worker.drain()).toBe(1);
1610 if (stage === "interrupt") {
1611 yield* TestClock.adjust("200 millis");
1612 expect(yield* worker.drain()).toBe(1);
1613 expect(yield* storedEffect).toMatchObject({ status: "pending", attemptCount: 3 });
1614 expect(harness.projection().runs.at(-1)?.status).toBe("starting");
1615 expect(harness.events).toEqual([]);
1616 expect(harness.writeIfRunCurrent).not.toHaveBeenCalled();
1617 } else {
1618 expect(yield* storedEffect).toMatchObject({ status: "succeeded", attemptCount: 2 });
1619 expect(harness.projection().runs.at(-1)?.status).toBe("failed");
1620 expect(harness.projection().turnItems).toMatchObject([
1621 { type: "error", title: "Turn failed to start" },
1622 ]);
1623 }
1624 }).pipe(
1625 Effect.provide(
1626 layerStartEffectWorker(harness, {
1627 interruptAndAwaitTerminal: () =>
1628 stage === "interrupt"
1629 ? Effect.fail(
1630 new ProviderTurnControlService.ProviderTurnControlError({
1631 threadId: run.threadId,
1632 operation: "restart",
1633 providerTurnId,
1634 cause: "provider interrupt failed",
1635 }),
1636 )
1637 : Effect.void,
1638 }),
1639 ),
1640 );
1641 },
1642 );
1643
1644 effectIt.effect.each(["Stop", "a replacement attempt"] as const)(
1645 "does not overwrite %s committed during the final start failure write",
1646 (change) => {
1647 const harness = makeLocalCommandHarness({ text: "Continue", missingProviderSession: true });
1648 const write = harness.writeIfRunCurrent.getMockImplementation()!;
1649 let protectedProjection = harness.projection();
1650 harness.writeIfRunCurrent.mockImplementationOnce((input) =>
1651 Effect.suspend(() => {
1652 const projection = harness.projection();
1653 if (change === "Stop") harness.interruptRun();
1654 else {
1655 harness.setProjection({
1656 ...projection,
1657 runs: projection.runs.map((run) => ({
1658 ...run,
1659 activeAttemptId: RunAttemptId.make("replacement-attempt"),
1660 rootNodeId: NodeId.make("replacement-root"),
1661 })),
1662 });
1663 }
1664 protectedProjection = harness.projection();
1665 return write(input);
1666 }),
1667 );
1668
1669 return Effect.gen(function* () {
1670 const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
1671 const { storedEffect } = yield* enqueueStart(harness.projection());
1672 yield* worker.drain();
1673 yield* TestClock.adjust("100 millis");
1674 expect(yield* worker.drain()).toBe(1);
1675 expect(harness.writeIfRunCurrent).toHaveBeenCalledOnce();
1676 expect(harness.projection()).toEqual(protectedProjection);
1677 expect(harness.events).toEqual([]);
1678 expect(yield* storedEffect).toMatchObject({ status: "succeeded" });
1679 }).pipe(Effect.provide(layerStartEffectWorker(harness)));
1680 },
1681 );
1682
1683 effectIt.effect("preserves a replacement committed after the final pre-running start read", () => {
1684 const harness = makeLocalCommandHarness({ text: "Continue", missingProviderSession: true });
1685 const read = harness.getTurnStartContext.getMockImplementation()!;
1686 let reads = 0;
1687 let replacement: OrchestrationV2ThreadProjection;
1688 harness.getTurnStartContext.mockImplementation(() =>
1689 Effect.suspend(() => {
1690 const snapshot = read();
1691 reads++;
1692 if (reads === 2) {
1693 const projection = harness.projection();
1694 const attemptId = RunAttemptId.make("replacement-before-fallback");
1695 const rootNodeId = NodeId.make("replacement-root-before-fallback");
1696 replacement = {
1697 ...projection,
1698 runs: projection.runs.map((run) => ({ ...run, activeAttemptId: attemptId, rootNodeId })),
1699 attempts: [
1700 ...projection.attempts,
1701 { ...projection.attempts[0]!, id: attemptId, rootNodeId },
1702 ],
1703 nodes: [...projection.nodes, { ...projection.nodes[0]!, id: rootNodeId }],
1704 };
1705 harness.setProjection(replacement);
1706 }
1707 return snapshot;
1708 }),
1709 );
1710 return Effect.gen(function* () {
1711 const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
1712 const { storedEffect } = yield* enqueueStart(harness.projection());
1713 yield* worker.drain();
1714 yield* TestClock.adjust("100 millis");
1715 expect(yield* worker.drain()).toBe(1);
1716 expect(harness.projection()).toEqual(replacement!);
1717 expect(harness.events).toEqual([]);
1718 expect(yield* storedEffect).toMatchObject({ status: "succeeded", attemptCount: 2 });
1719 }).pipe(Effect.provide(layerStartEffectWorker(harness)));
1720 });
1721
1722 effectIt.effect("preserves a replacement committed before exhausted start settlement reads", () =>
1723 Effect.gen(function* () {
1724 const harness = makeLocalCommandHarness({ text: "Continue", failReadsAfterRunning: true });
1725 harness.getRuntimeRecoveryProjection.mockImplementation(() =>
1726 Effect.succeed(harness.projection()),
1727 );
1728 harness.getRun.mockImplementation((_threadId, id) =>
1729 Effect.succeed(harness.projection().runs.find((candidate) => candidate.id === id)),
1730 );
1731 const projection = harness.projection();
1732 const run = projection.runs[0]!;
1733 const replacementAttemptId = RunAttemptId.make("replacement-before-settlement");
1734 const replacementRootId = NodeId.make("replacement-root-before-settlement");
1735 const providerTurn = {
1736 id: ProviderTurnId.make("old-start-provider-turn"),
1737 providerThreadId: run.providerThreadId!,
1738 nodeId: run.rootNodeId!,
1739 runAttemptId: run.activeAttemptId!,
1740 nativeTurnRef: null,
1741 ordinal: 1,
1742 status: "running" as const,
1743 startedAt: projection.updatedAt,
1744 completedAt: null,
1745 };
1746 const turnIngested = yield* Deferred.make<void>();
1747 const storageFailure = new EventSink.EventSinkWriteError({
1748 eventCount: 5,
1749 cause: "terminal-write ownership query unavailable",
1750 });
1751 const write = harness.writeIfRunCurrent.getMockImplementation()!;
1752 harness.writeIfRunCurrent.mockImplementation((input) =>
1753 input.expectedStatus === "running" ? Effect.fail(storageFailure) : write(input),
1754 );
1755 harness.open.mockImplementationOnce(() =>
1756 Effect.fail(
1757 new ProviderSessionManager.ProviderSessionOpenError({
1758 instanceId: harness.newInstanceId,
1759 providerSessionId: projection.providerThreads[0]!.providerSessionId!,
1760 cause: "transient session failure",
1761 }),
1762 ),
1763 );
1764 const executionLayer = RunExecutionService.layer.pipe(
1765 Layer.provide(
1766 Layer.mergeAll(
1767 McpAppModelContext.layerEmpty,
1768 PreviewAgentActivity.layer,
1769 Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }),
1770 Layer.mock(TurnItemPositionStore.TurnItemPositionStoreV2)({}),
1771 Layer.mock(EventSink.EventSinkV2)({
1772 consumeProviderEvents: (source, process, after) =>
1773 source.pipe(
1774 Stream.mapEffect((event) => process(event).pipe(Effect.andThen(after))),
1775 Stream.takeUntil(Boolean),
1776 Stream.runDrain,
1777 ),
1778 writeIfRunCurrent: harness.writeIfRunCurrent,
1779 }),
1780 IdAllocator.layer,
1781 ThreadCommandExecutor.layer,
1782 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
1783 ingestNormalized: () =>
1784 Effect.sync(() => {
1785 harness.setProjection({ ...harness.projection(), providerTurns: [providerTurn] });
1786 }).pipe(Effect.andThen(Deferred.succeed(turnIngested, undefined)), Effect.as([])),
1787 }),
1788 ServerSettings.layerTest(),
1789 ),
1790 ),
1791 );
1792 let protectedProjection = projection;
1793 let escapedStartError: RunExecutionService.RunExecutionServiceV2Error | undefined;
1794 return yield* Effect.gen(function* () {
1795 const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
1796 const outbox = yield* EffectOutbox.EffectOutboxV2;
1797 harness.startRootRun.mockImplementation((input) =>
1798 Effect.gen(function* () {
1799 const execution = yield* RunExecutionService.RunExecutionServiceV2;
1800 yield* execution.startRootRun({
1801 ...input,
1802 session: {
1803 ...input.session,
1804 events: Stream.concat(
1805 Stream.succeed({
1806 type: "provider_turn.updated" as const,
1807 driver: input.providerThread.driver,
1808 threadId: run.threadId,
1809 providerTurn,
1810 }),
1811 Stream.never,
1812 ),
1813 startTurn: () =>
1814 Effect.gen(function* () {
1815 yield* Deferred.await(turnIngested);
1816 const current = harness.projection();
1817 expect(current.runs[0]?.status).toBe("running");
1818 // Steering commits a new attempt while the original start still owns the lane.
1819 harness.setProjection({
1820 ...current,
1821 runs: [
1822 {
1823 ...current.runs[0]!,
1824 status: "starting",
1825 activeAttemptId: replacementAttemptId,
1826 rootNodeId: replacementRootId,
1827 },
1828 ],
1829 attempts: [
1830 {
1831 ...current.attempts[0]!,
1832 status: "superseded",
1833 completedAt: current.updatedAt,
1834 },
1835 {
1836 ...projection.attempts[0]!,
1837 id: replacementAttemptId,
1838 rootNodeId: replacementRootId,
1839 attemptOrdinal: 2,
1840 reason: "steering_restart",
1841 },
1842 ],
1843 nodes: [
1844 {
1845 ...current.nodes[0]!,
1846 status: "interrupted",
1847 completedAt: current.updatedAt,
1848 },
1849 {
1850 ...projection.nodes[0]!,
1851 id: replacementRootId,
1852 rootNodeId: replacementRootId,
1853 },
1854 ],
1855 });
1856 protectedProjection = harness.projection();
1857 yield* outbox
1858 .enqueue([
1859 {
1860 id: "effect:replacement-restart",
1861 commandId: CommandId.make("command:replacement-restart"),
1862 threadId: run.threadId,
1863 request: {
1864 type: "provider-turn.restart",
1865 runId: run.id,
1866 providerSessionId: input.providerSessionId,
1867 providerThreadId: run.providerThreadId!,
1868 providerTurnId: providerTurn.id,
1869 interruptedAttemptId: input.attemptId,
1870 },
1871 },
1872 ])
1873 .pipe(Effect.orDie);
1874 return yield* new ProviderAdapter.ProviderAdapterTurnStartError({
1875 driver: input.session.driver,
1876 threadId: run.threadId,
1877 providerThreadId: run.providerThreadId!,
1878 runId: run.id,
1879 cause: "provider start failed after publishing its turn",
1880 });
1881 }),
1882 },
1883 });
1884 }).pipe(
1885 Effect.tapError((error) =>
1886 Effect.sync(() => {
1887 escapedStartError = error;
1888 }),
1889 ),
1890 Effect.provide(executionLayer.pipe(Layer.orDie)),
1891 ),
1892 );
1893 const { storedEffect } = yield* enqueueStart(projection);
1894 expect(yield* worker.drain()).toBe(1);
1895 yield* TestClock.adjust("100 millis");
1896 expect(yield* worker.drain(1)).toBe(1);
1897 expect(escapedStartError).toMatchObject({ cause: { write: storageFailure } });
1898 expect(yield* outbox.get("effect:replacement-restart")).toMatchObject({
1899 value: { status: "pending", attemptCount: 0 },
1900 });
1901 expect(yield* storedEffect).toMatchObject({ status: "succeeded", attemptCount: 2 });
1902 expect(harness.projection()).toEqual(protectedProjection);
1903 }).pipe(Effect.provide(layerStartEffectWorker(harness)));
1904 }),
1905 );
1906
1907 effectIt.effect(
1908 "recovers the local or pushed thread tip, else the current branch, remote default, or detached HEAD",
1909 () =>
1910 Effect.gen(function* () {
1911 const fileSystem = yield* FileSystem.FileSystem;
1912 const path = yield* Path.Path;
1913 const gitWorkflow = yield* GitWorkflow.GitWorkflowService;
1914 const driver = yield* GitVcsDriver.GitVcsDriver;
1915 const root = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-recreate-repo-" });
1916 const workspaceRoot = path.join(root, "project");
1917 const originRoot = path.join(root, "origin.git");
1918 const worktreesDir = path.join(root, "worktrees");
1919 const worktreePath = path.join(worktreesDir, "project", "restore");
1920 yield* fileSystem.makeDirectory(workspaceRoot);
1921 const git = (cwd: string, args: ReadonlyArray<string>) =>
1922 driver
1923 .execute({ operation: "test.worktreeRecovery", cwd, args })
1924 .pipe(Effect.map((result) => result.stdout.trim()));
1925 const branchConfig = (key: string) =>
1926 git(workspaceRoot, ["config", "--get", `branch.feature/restore.${key}`]).pipe(
1927 Effect.orElseSucceed(() => ""),
1928 );
1929 yield* git(root, ["init", "--bare", "-b", "main", originRoot]);
1930 yield* git(workspaceRoot, ["init", "-b", "main"]);
1931 yield* git(workspaceRoot, ["config", "user.name", "Test"]);
1932 yield* git(workspaceRoot, ["config", "user.email", "test@example.com"]);
1933 yield* git(workspaceRoot, ["commit", "--allow-empty", "-m", "base"]);
1934 const baseTip = yield* git(workspaceRoot, ["rev-parse", "HEAD"]);
1935 yield* git(workspaceRoot, ["remote", "add", "origin", originRoot]);
1936 yield* git(workspaceRoot, ["push", "origin", "main"]);
1937 yield* git(workspaceRoot, ["remote", "set-head", "origin", "main"]);
1938 yield* git(workspaceRoot, ["worktree", "add", "-b", "feature/restore", worktreePath]);
1939 yield* fileSystem.writeFileString(path.join(worktreePath, "thread.txt"), "thread work");
1940 yield* git(worktreePath, ["add", "."]);
1941 yield* git(worktreePath, ["commit", "-m", "thread tip"]);
1942 const threadTip = yield* git(worktreePath, ["rev-parse", "HEAD"]);
1943 yield* git(workspaceRoot, ["switch", "-c", "project-current"]);
1944 yield* fileSystem.writeFileString(
1945 path.join(workspaceRoot, "project.txt"),
1946 "new project code",
1947 );
1948 yield* git(workspaceRoot, ["add", "."]);
1949 yield* git(workspaceRoot, ["commit", "-m", "project tip"]);
1950 const projectTip = yield* git(workspaceRoot, ["rev-parse", "HEAD"]);
1951 expect(projectTip).not.toBe(threadTip);
1952 // Tags must neither hide the thread branch nor select a different recovery tip.
1953 yield* git(workspaceRoot, ["tag", "feature/restore", projectTip]);
1954 yield* git(workspaceRoot, ["tag", "project-current", threadTip]);
1955 yield* git(workspaceRoot, ["tag", "origin/feature/restore", threadTip]);
1956
1957 const expectedTips = {
1958 retained: { tip: threadTip, file: "thread.txt", upstream: null, base: "" },
1959 // A failed removal can leave the folder behind with only dependencies inside.
1960 leftover: { tip: threadTip, file: "thread.txt", upstream: null, base: "" },
1961 deleted: { tip: projectTip, file: "project.txt", upstream: null, base: "project-current" },
1962 pushed: {
1963 tip: threadTip,
1964 file: "thread.txt",
1965 upstream: "refs/remotes/origin/feature/restore",
1966 base: "",
1967 },
1968 "pushed-detached": {
1969 tip: threadTip,
1970 file: "thread.txt",
1971 upstream: "refs/remotes/origin/feature/restore",
1972 base: "",
1973 },
1974 "pushed-sole-remote": {
1975 tip: threadTip,
1976 file: "thread.txt",
1977 upstream: "refs/remotes/upstream/feature/restore",
1978 base: "",
1979 },
1980 detached: { tip: baseTip, file: null, upstream: null, base: "main" },
1981 "detached-no-remote": { tip: projectTip, file: "project.txt", upstream: null, base: "" },
1982 } as const;
1983 // Reuse the workflow across turns so recovery cannot select a cached project branch.
1984 for (const scenario of Object.keys(expectedTips) as Array<keyof typeof expectedTips>) {
1985 const expected = expectedTips[scenario];
1986 if (scenario === "pushed") {
1987 yield* git(workspaceRoot, ["push", "origin", `${threadTip}:refs/heads/feature/restore`]);
1988 yield* git(workspaceRoot, ["fetch", "origin"]);
1989 }
1990 if (scenario === "pushed-detached") {
1991 yield* git(workspaceRoot, ["switch", "--detach"]);
1992 expect(yield* git(workspaceRoot, ["branch", "--show-current"])).toBe("");
1993 }
1994 if (scenario === "pushed-sole-remote") {
1995 yield* git(workspaceRoot, ["remote", "rename", "origin", "upstream"]);
1996 }
1997 if (scenario === "detached") {
1998 yield* git(workspaceRoot, ["remote", "rename", "upstream", "origin"]);
1999 yield* git(workspaceRoot, ["push", "origin", "--delete", "feature/restore"]);
2000 }
2001 if (scenario === "detached-no-remote") {
2002 yield* git(workspaceRoot, ["remote", "remove", "origin"]);
2003 }
2004 yield* fileSystem.remove(worktreePath, { recursive: true });
2005 if (scenario === "leftover") {
2006 yield* fileSystem.makeDirectory(path.join(worktreePath, "node_modules"), {
2007 recursive: true,
2008 });
2009 }
2010 if (scenario !== "retained" && scenario !== "leftover") {
2011 yield* git(workspaceRoot, ["worktree", "prune"]);
2012 yield* git(workspaceRoot, ["branch", "-D", "feature/restore"]);
2013 }
2014 const harness = makeLocalCommandHarness({
2015 text: "Continue",
2016 workspaceRoot,
2017 worktreePath,
2018 worktreesDir,
2019 fileSystem,
2020 gitWorkflow,
2021 failReadsAfterRunning: true,
2022 });
2023 const open = harness.open.getMockImplementation()!;
2024 harness.open.mockImplementation(() =>
2025 Effect.gen(function* () {
2026 expect(yield* git(worktreePath, ["branch", "--show-current"])).toBe("feature/restore");
2027 expect(yield* git(worktreePath, ["rev-parse", "HEAD"])).toBe(expected.tip);
2028 for (const file of ["thread.txt", "project.txt"]) {
2029 expect(yield* fileSystem.exists(path.join(worktreePath, file))).toBe(
2030 file === expected.file,
2031 );
2032 }
2033 expect(yield* branchConfig("merge")).toBe(
2034 expected.upstream === null ? "" : "refs/heads/feature/restore",
2035 );
2036 if (expected.upstream !== null) {
2037 expect(
2038 yield* git(worktreePath, ["rev-parse", "--symbolic-full-name", "@{upstream}"]),
2039 ).toBe(expected.upstream);
2040 }
2041 expect(yield* branchConfig("t3-merge-base")).toBe(expected.base);
2042 return yield* open();
2043 }).pipe(Effect.orDie),
2044 );
2045
2046 yield* harness.start;
2047
2048 expect(harness.open).toHaveBeenCalledOnce();
2049 expect(harness.startRootRun).toHaveBeenCalledWith(
2050 expect.objectContaining({
2051 runtimePolicy: expect.objectContaining({ cwd: worktreePath }),
2052 }),
2053 );
2054 expect(harness.projection().runs.at(-1)?.status).toBe("running");
2055 }
2056 }).pipe(Effect.scoped, Effect.provide(layerRealGit)),
2057 );
2058
2059 effectIt.effect("leaves a managed worktree folder without .git that holds user files", () =>
2060 Effect.gen(function* () {
2061 const fileSystem = yield* FileSystem.FileSystem;
2062 const path = yield* Path.Path;
2063 const worktreesDir = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-leftover-" });
2064 const worktreePath = path.join(worktreesDir, "project", "restore");
2065 yield* fileSystem.makeDirectory(path.join(worktreePath, "node_modules"), { recursive: true });
2066 yield* fileSystem.writeFileString(path.join(worktreePath, "notes.md"), "keep me");
2067 const harness = makeLocalCommandHarness({
2068 text: "Continue",
2069 worktreePath,
2070 worktreesDir,
2071 fileSystem,
2072 failReadsAfterRunning: true,
2073 });
2074 yield* harness.start;
2075 expect(harness.createWorktree).not.toHaveBeenCalled();
2076 expect(yield* fileSystem.readFileString(path.join(worktreePath, "notes.md"))).toBe("keep me");
2077 }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
2078 );
2079
2080 effectIt.effect(
2081 "preserves recreation interrupts and retries, then fails the final attempt before provider start",
2082 () =>
2083 Effect.gen(function* () {
2084 const fileSystem = yield* FileSystem.FileSystem;
2085 const path = yield* Path.Path;
2086 const worktreesDir = yield* fileSystem.makeTempDirectoryScoped({
2087 prefix: "t3-recreate-failure-",
2088 });
2089 const worktreePath = path.join(worktreesDir, "project", "restore");
2090 const harness = makeLocalCommandHarness({
2091 text: "Continue",
2092 worktreesDir,
2093 worktreePath,
2094 fileSystem,
2095 failReadsAfterRunning: true,
2096 });
2097 harness.createWorktree.mockImplementationOnce(() => Effect.interrupt);
2098 harness.createWorktree.mockImplementation(() =>
2099 Effect.fail(
2100 new GitCommandError({
2101 operation: "worktree-add",
2102 command: "git worktree add",
2103 cwd: "/tmp/native-account-project",
2104 detail: "fatal: private git diagnostic",
2105 }),
2106 ),
2107 );
2108
2109 const interrupted = yield* Effect.exit(harness.start);
2110 expect(interrupted._tag).toBe("Failure");
2111 if (interrupted._tag === "Failure")
2112 expect(Cause.hasInterruptsOnly(interrupted.cause)).toBe(true);
2113 expect(harness.events).toEqual([]);
2114 expect(harness.projection().runs.at(-1)?.status).toBe("starting");
2115
2116 const retry = yield* Effect.result(harness.startWithRetry);
2117 expect(retry).toMatchObject({ _tag: "Failure", failure: { _tag: "ProviderTurnStartError" } });
2118 expect(harness.events).toEqual([]);
2119 expect(harness.projection().runs.at(-1)?.status).toBe("starting");
2120
2121 yield* harness.start;
2122 expect(harness.createWorktree).toHaveBeenCalledTimes(3);
2123 expect(harness.writeIfRunCurrent).toHaveBeenCalledOnce();
2124 expect(harness.open).not.toHaveBeenCalled();
2125 expect(harness.startRootRun).not.toHaveBeenCalled();
2126 const projection = harness.projection();
2127 expect(projection.runs.at(-1)).toMatchObject({ status: "failed", startedAt: null });
2128 expect(projection.attempts[0]).toMatchObject({ status: "failed", startedAt: null });
2129 expect(projection.nodes[0]).toMatchObject({ status: "failed", startedAt: null });
2130 expect(projection.turnItems).toMatchObject([
2131 {
2132 type: "error",
2133 failure: {
2134 message: `Could not recreate the worktree for branch 'feature/restore' at ${worktreePath}.`,
2135 },
2136 },
2137 ]);
2138 expect(JSON.stringify(harness.events)).not.toContain("private git diagnostic");
2139 }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
2140 );
2141
2142 effectIt.effect("recovers an abandoned failed turn before continuing its native conversation", () =>
2143 Effect.gen(function* () {
2144 const harness = makeLocalCommandHarness({
2145 text: "Continue",
2146 previousTurn: { runStatus: "failed", turnStatus: "running" },
2147 });
2148
2149 yield* harness.start;
2150
2151 expect(harness.interruptTurn).toHaveBeenCalledOnce();
2152 expect(harness.projection().runs.at(-1)?.status).toBe("running");
2153 expect(harness.projection().providerTurns[0]).toMatchObject({
2154 status: "interrupted",
2155 completedAt: expect.anything(),
2156 });
2157 expect(harness.startRootRun).toHaveBeenCalledOnce();
2158 expect(harness.startRootRun.mock.calls[0]?.[0].providerThread.nativeThreadRef?.nativeId).toBe(
2159 "existing-native-conversation",
2160 );
2161 }),
2162 );
2163
2164 effectIt.effect(
2165 "recovers a running turn its interrupted run left behind on a superseded attempt",
2166 () =>
2167 Effect.gen(function* () {
2168 const harness = makeLocalCommandHarness({
2169 text: "Continue",
2170 previousTurn: { runStatus: "interrupted", turnStatus: "running", supersededTurn: true },
2171 });
2172
2173 yield* harness.start;
2174
2175 expect(harness.interruptTurn).toHaveBeenCalledOnce();
2176 // The steered attempt's turn was already interrupted in the fixture; the
2177 // repair this asserts is the running turn left on the superseded attempt.
2178 const abandoned = harness
2179 .projection()
2180 .providerTurns.find((turn) => turn.id === ProviderTurnId.make("previous-provider-turn"));
2181 expect(abandoned).toMatchObject({
2182 status: "interrupted",
2183 completedAt: expect.anything(),
2184 });
2185 expect(harness.projection().runs.at(-1)?.status).toBe("running");
2186 expect(harness.startRootRun).toHaveBeenCalledOnce();
2187 }),
2188 );
2189
2190 it("finds abandoned turns in a long history reading each record a bounded number of times", () => {
2191 let reads = 0;
2192 const counted = <T extends object>(record: T): T =>
2193 new Proxy(record, {
2194 get: (target, key, receiver) => {
2195 reads += 1;
2196 return Reflect.get(target, key, receiver);
2197 },
2198 });
2199 const providerThreadId = ProviderThreadId.make("provider-thread-long-history");
2200 const endings = ["failed", "interrupted", "cancelled"] as const;
2201 const runs = [];
2202 const attempts = [];
2203 const providerTurns = [];
2204 for (let index = 0; index < 2_000; index += 1) {
2205 // Every tenth run ended without completing, after a steering restart.
2206 const status = index % 10 === 0 ? endings[(index / 10) % endings.length]! : "completed";
2207 const attemptIds = status === "completed" ? [`a${index}`] : [`a${index}-steered`, `a${index}`];
2208 runs.push({
2209 id: `r${index}`,
2210 ordinal: index + 1,
2211 status,
2212 providerThreadId,
2213 activeAttemptId: attemptIds.at(-1),
2214 });
2215 for (const attemptId of attemptIds) {
2216 attempts.push({ id: attemptId, runId: `r${index}` });
2217 const running = attemptId === "a10-steered" || attemptId === "a20" || attemptId === "a21";
2218 providerTurns.push({
2219 id: `t-${attemptId}`,
2220 providerThreadId,
2221 runAttemptId: attemptId,
2222 status: running ? "running" : "completed",
2223 });
2224 }
2225 }
2226 const projection = {
2227 runs: runs.map(counted),
2228 attempts: attempts.map(counted),
2229 providerTurns: providerTurns.map(counted),
2230 } as unknown as Parameters<typeof ProviderTurnStart.abandonedProviderTurns>[0];
2231
2232 const abandoned = ProviderTurnStart.abandonedProviderTurns(
2233 projection,
2234 { ordinal: 2_001 },
2235 providerThreadId,
2236 );
2237
2238 expect(abandoned.map(({ turn, run }) => [turn.id, run.id])).toEqual([
2239 ["t-a10-steered", "r10"],
2240 ["t-a20", "r20"],
2241 ]);
2242 const records = runs.length + attempts.length + providerTurns.length;
2243 expect(reads).toBeLessThanOrEqual(4 * records);
2244 });
2245
2246 effectIt.effect("sends new steering text when restarting a manual continuation", () =>
2247 Effect.gen(function* () {
2248 const harness = makeLocalCommandHarness({
2249 text: "New steering instruction",
2250 manualContinuation: true,
2251 steeringRestart: true,
2252 nativeResumeUnsupported: true,
2253 sideChat: { contextRun: "delivered" },
2254 });
2255 yield* harness.start;
2256 expect(harness.startTurn.mock.calls[0]?.[0].message.text).toBe("New steering instruction");
2257 expect(harness.startRootRun.mock.calls[0]?.[0].resume).toBe(false);
2258 }),
2259 );
2260
2261 effectIt.effect("refuses unsupported resume instead of manufacturing a user prompt", () =>
2262 Effect.gen(function* () {
2263 const harness = makeLocalCommandHarness({
2264 text: "",
2265 manualContinuation: true,
2266 nativeResumeUnsupported: true,
2267 sideChat: { contextRun: "delivered" },
2268 });
2269 yield* harness.start;
2270 expect(harness.startTurn).not.toHaveBeenCalled();
2271 expect(harness.startRootRun).not.toHaveBeenCalled();
2272 expect(harness.settledRuns).toEqual(["failed"]);
2273 expect(harness.projection().turnItems).toEqual(
2274 expect.arrayContaining([
2275 expect.objectContaining({
2276 type: "error",
2277 failure: expect.objectContaining({
2278 message:
2279 "This provider cannot resume without a new message. Send a message to continue.",
2280 }),
2281 }),
2282 ]),
2283 );
2284 }),
2285 );
2286
2287 effectIt.effect.each([
2288 { contextRun: "current", runtimeMode: "full-access", context: "SIDE_CHAT_CONTEXT" },
2289 { contextRun: "failed", runtimeMode: "full-access", context: "SIDE_CHAT_CONTEXT" },
2290 { contextRun: "cancelled", runtimeMode: "full-access", context: "SIDE_CHAT_CONTEXT" },
2291 {
2292 contextRun: "failed",
2293 switchedProvider: true,
2294 runtimeMode: "full-access",
2295 context: "SIDE_CHAT_CONTEXT",
2296 },
2297 { contextRun: "delivered", runtimeMode: "full-access", context: "" },
2298 { contextRun: "delivered", runtimeMode: "full-access", context: "", unsentResume: true },
2299 {
2300 contextRun: "delivered",
2301 runtimeMode: "full-access",
2302 context: "",
2303 unsentResume: true,
2304 repeatedResume: true,
2305 },
2306 {
2307 contextRun: "delivered",
2308 runtimeMode: "full-access",
2309 context: "",
2310 unsentResume: true,
2311 acceptedResume: true,
2312 },
2313 {
2314 contextRun: "delivered",
2315 runtimeMode: "full-access",
2316 context: "",
2317 unsentResume: true,
2318 acceptedResume: true,
2319 restartedResume: true,
2320 },
2321 {
2322 contextRun: "current",
2323 threadRuntimeMode: "approval-required",
2324 runtimeMode: "approval-required",
2325 context: "SIDE_CHAT_CONTEXT",
2326 },
2327 ] as const)(
2328 "sends side chat context inline once, for the first turn that reaches the provider: %j",
2329 ({ runtimeMode, context, ...sideChat }) =>
2330 Effect.gen(function* () {
2331 const harness = makeLocalCommandHarness({
2332 text: "",
2333 manualContinuation: true,
2334 sideChat,
2335 });
2336
2337 yield* harness.start;
2338
2339 expect(harness.open).toHaveBeenCalledWith(
2340 expect.objectContaining({ runtimePolicy: expect.objectContaining({ runtimeMode }) }),
2341 );
2342 const resendsOriginal =
2343 "unsentResume" in sideChat &&
2344 (!("acceptedResume" in sideChat) || "restartedResume" in sideChat);
2345 const texts = harness.startTurn.mock.calls.map(([turn]) => turn.message.text);
2346 expect(texts).toHaveLength(1);
2347 if (resendsOriginal || context === "") {
2348 expect(texts[0]).toBe(
2349 withInlineContext(context, resendsOriginal ? "Original request" : ""),
2350 );
2351 } else {
2352 // A resume that has to carry context still tells the agent to continue.
2353 const contextPrefix = withInlineContext(context, "");
2354 expect(texts[0]?.startsWith(contextPrefix)).toBe(true);
2355 expect(texts[0]?.slice(contextPrefix.length).trim()).not.toBe("");
2356 }
2357 expect(harness.startTurn.mock.calls[0]?.[0].resume === true).toBe(
2358 context === "" && !resendsOriginal,
2359 );
2360 expect(harness.projection().contextHandoffs).toMatchObject([
2361 {
2362 id: "side-chat-context",
2363 toProviderThreadId: "new-provider-thread",
2364 delivery: { nativeThreadId: "side-chat-native", status: "inline" },
2365 },
2366 ]);
2367 }),
2368 );
2369
2370 effectIt.effect.each([
2371 { scheduled: false, status: "superseded" },
2372 { scheduled: true, status: "ready" },
2373 ] as const)(
2374 "a fresh run starts without failed runs' side chat notes; an unscheduled one retires them: %j",
2375 ({ scheduled, status }) =>
2376 Effect.gen(function* () {
2377 const harness = makeLocalCommandHarness({
2378 text: "Start over",
2379 sideChat: { contextRun: "failed" },
2380 });
2381 const projection = harness.projection();
2382 const current = projection.runs.at(-1)!;
2383 const [note] = projection.contextHandoffs;
2384 harness.setProjection({
2385 ...projection,
2386 runs: projection.runs.map((run) =>
2387 run.id === current.id ? { ...run, freshContext: true } : run,
2388 ),
2389 // Each send that fails before the side chat's fork resolves leaves its own note.
2390 contextHandoffs: [
2391 ...projection.contextHandoffs,
2392 { ...note!, id: ContextHandoffId.make("side-chat-context-retry") },
2393 ],
2394 turnItems: scheduled
2395 ? [
2396 ...projection.turnItems,
2397 {
2398 type: "user_message",
2399 id: TurnItemId.make("scheduled-fresh-message"),
2400 threadId: projection.thread.id,
2401 runId: current.id,
2402 nodeId: null,
2403 providerThreadId: current.providerThreadId,
2404 providerTurnId: null,
2405 nativeItemRef: null,
2406 parentItemId: null,
2407 ordinal: 0,
2408 status: "completed",
2409 title: null,
2410 startedAt: projection.updatedAt,
2411 completedAt: projection.updatedAt,
2412 updatedAt: projection.updatedAt,
2413 createdBy: "user",
2414 creationSource: "server",
2415 messageId: current.userMessageId,
2416 text: "Start over",
2417 inputIntent: "turn_start",
2418 attachments: [],
2419 scheduledTaskId: ScheduledTaskId.make("fresh-session-task"),
2420 },
2421 ]
2422 : projection.turnItems,
2423 });
2424
2425 yield* harness.start;
2426
2427 expect(harness.startTurn.mock.calls[0]?.[0].message.text).toBe("Start over");
2428 expect(
2429 harness.projection().contextHandoffs.map((handoff) => [handoff.status, handoff.delivery]),
2430 ).toEqual([
2431 [status, undefined],
2432 [status, undefined],
2433 ]);
2434 }),
2435 );
2436
2437 effectIt.effect.each([
2438 {
2439 nativeForkFrom: "idle",
2440 canForkWhileSourceRunning: false,
2441 sourceBoundary: "native",
2442 forks: true,
2443 transferStatus: "consumed",
2444 handoffIds: [],
2445 },
2446 {
2447 nativeForkFrom: "busy",
2448 canForkWhileSourceRunning: true,
2449 sourceBoundary: "native",
2450 forks: true,
2451 transferStatus: "consumed",
2452 handoffIds: [],
2453 },
2454 {
2455 nativeForkFrom: "busy",
2456 canForkWhileSourceRunning: false,
2457 sourceBoundary: "native",
2458 forks: false,
2459 transferStatus: "resolved_portable",
2460 handoffIds: ["side-chat-fork-context"],
2461 },
2462 {
2463 nativeForkFrom: "busy",
2464 canForkWhileSourceRunning: true,
2465 sourceBoundary: "synthetic",
2466 forks: false,
2467 transferStatus: "resolved_portable",
2468 handoffIds: ["side-chat-fork-context"],
2469 },
2470 {
2471 nativeForkFrom: "busy",
2472 canForkWhileSourceRunning: true,
2473 exposesNativeTurnId: false,
2474 sourceBoundary: "synthetic",
2475 forks: true,
2476 transferStatus: "consumed",
2477 handoffIds: [],
2478 },
2479 ] as const)(
2480 "runs a planned native side chat fork when the parent is $nativeForkFrom with canForkWhileSourceRunning=$canForkWhileSourceRunning, sourceBoundary=$sourceBoundary and exposesNativeTurnId=$exposesNativeTurnId",
2481 ({ nativeForkFrom, sourceBoundary, forks, transferStatus, handoffIds, ...capability }) =>
2482 Effect.gen(function* () {
2483 const harness = makeLocalCommandHarness({
2484 text: "What is the main thread doing?",
2485 sideChat: { contextRun: "current", nativeForkFrom, sourceBoundary },
2486 ...capability,
2487 });
2488
2489 yield* harness.start;
2490
2491 expect(harness.forkThread).toHaveBeenCalledTimes(forks ? 1 : 0);
2492 // The parent's rolled-back turn is gone from its native conversation.
2493 expect(
2494 harness.forkThread.mock.calls[0]?.[0].sourceProviderTurns?.map((turn) => turn.id),
2495 ).toEqual(forks ? ["side-chat-parent-turn-1"] : undefined);
2496 if (forks) {
2497 expect(harness.forkThread).toHaveBeenCalledWith(
2498 expect.objectContaining({ providerTurnId: "side-chat-parent-turn-1" }),
2499 );
2500 }
2501 expect(harness.projection().contextTransfers).toMatchObject([
2502 { id: "side-chat-fork", status: transferStatus },
2503 ]);
2504 expect(harness.startTurn.mock.calls[0]?.[0].message.text.includes("PARENT_HISTORY")).toBe(
2505 !forks,
2506 );
2507 expect(harness.projection().providerThreads[0]?.handoffIds).toEqual(handoffIds);
2508 }),
2509 );
2510
2511 effectIt.effect.each(
2512 (
2513 [
2514 { goalDriver: "opencode", nativeGoals: false },
2515 { goalDriver: "opencode", nativeGoals: true },
2516 { goalDriver: "antigravity", nativeGoals: false },
2517 { goalDriver: "antigravity", nativeGoals: true },
2518 ] as const
2519 ).flatMap((choice) => [
2520 { ...choice, resuming: false },
2521 { ...choice, resuming: true },
2522 ]),
2523 )(
2524 "$goalDriver goal prompts defer switched-provider history only with native support: $nativeGoals, resuming: $resuming",
2525 ({ goalDriver, nativeGoals, resuming }) =>
2526 Effect.gen(function* () {
2527 const harness = makeLocalCommandHarness({
2528 text: resuming ? "" : "/goal all tests pass",
2529 manualContinuation: resuming,
2530 goalDriver,
2531 nativeGoals,
2532 sideChat: { contextRun: "failed", switchedProvider: true, unsentResume: resuming },
2533 });
2534 if (resuming) {
2535 const projection = harness.projection();
2536 harness.setProjection({
2537 ...projection,
2538 messages: projection.messages.map((message) =>
2539 message.id === "original-unsent-message"
2540 ? { ...message, text: "/goal all tests pass" }
2541 : message,
2542 ),
2543 });
2544 }
2545 harness.tryHandlePromptCommand.mockReturnValue(Effect.succeed(false));
2546 yield* harness.start;
2547 expect(harness.startTurn.mock.calls[0]?.[0].message.text.includes("SIDE_CHAT_CONTEXT")).toBe(
2548 !nativeGoals,
2549 );
2550 if (nativeGoals)
2551 expect(harness.startTurn.mock.calls[0]?.[0].message.text).toBe("/goal all tests pass");
2552 }),
2553 );
2554
2555 effectIt.effect.each(["before-start", "during-fork"] as const)(
2556 "recovers a planned native fork after the parent rolls back %s",
2557 (rollback) =>
2558 Effect.gen(function* () {
2559 const harness = makeLocalCommandHarness({
2560 text: "Continue from the selected turn",
2561 sideChat: {
2562 contextRun: "current",
2563 nativeForkFrom: "idle",
2564 sourceRolledBack: rollback === "before-start",
2565 },
2566 });
2567 const parentThreadId = ThreadId.make("side-chat-parent");
2568 let parent = {
2569 ...harness.projection(),
2570 ...(yield* harness.getThreadRecords(parentThreadId, [
2571 "runs",
2572 "attempts",
2573 "providerThreads",
2574 "providerTurns",
2575 "turnItems",
2576 ])),
2577 };
2578 const selectedRun = parent.runs[0]!;
2579 const earlierRun = {
2580 ...selectedRun,
2581 id: RunId.make("side-chat-kept-run"),
2582 ordinal: 1,
2583 activeAttemptId: null,
2584 status: "completed" as const,
2585 };
2586 const items = [earlierRun, selectedRun].map((sourceRun, ordinal) => ({
2587 type: "user_message" as const,
2588 id: TurnItemId.make(`parent-item-${ordinal}`),
2589 threadId: parentThreadId,
2590 runId: sourceRun.id,
2591 nodeId: null,
2592 providerThreadId: sourceRun.providerThreadId,
2593 providerTurnId: null,
2594 nativeItemRef: null,
2595 parentItemId: null,
2596 ordinal,
2597 status: "completed" as const,
2598 title: null,
2599 startedAt: parent.updatedAt,
2600 completedAt: parent.updatedAt,
2601 updatedAt: parent.updatedAt,
2602 createdBy: "user" as const,
2603 creationSource: "server" as const,
2604 messageId: MessageId.make(`parent-message-${ordinal}`),
2605 text: ordinal === 0 ? "KEPT_HISTORY" : "ROLLED_BACK_HISTORY",
2606 inputIntent: "turn_start" as const,
2607 attachments: [],
2608 }));
2609 parent = {
2610 ...parent,
2611 runs: [
2612 earlierRun,
2613 ...parent.runs.map((sourceRun) => ({ ...sourceRun, ordinal: sourceRun.ordinal + 1 })),
2614 ],
2615 turnItems: items,
2616 };
2617 harness.getThreadRecords.mockImplementation((id) =>
2618 Effect.succeed(id === parentThreadId ? parent : harness.projection()),
2619 );
2620 harness.forkThread.mockImplementation(() =>
2621 Effect.sync(() => {
2622 parent = {
2623 ...parent,
2624 runs: parent.runs.map((sourceRun) =>
2625 sourceRun.id === selectedRun.id ? { ...sourceRun, status: "rolled_back" } : sourceRun,
2626 ),
2627 };
2628 }).pipe(
2629 Effect.andThen(
2630 Effect.fail(
2631 new ProviderAdapter.ProviderAdapterForkThreadError({
2632 driver: ProviderDriverKind.make("codex"),
2633 providerThreadId: ProviderThreadId.make("side-chat-parent-provider-thread"),
2634 cause: "Selected native turn was removed by rollback",
2635 }),
2636 ),
2637 ),
2638 ),
2639 );
2640
2641 yield* harness.start;
2642
2643 expect(harness.forkThread).toHaveBeenCalledTimes(rollback === "before-start" ? 0 : 1);
2644 expect(harness.projection().contextTransfers).toMatchObject([
2645 { id: "side-chat-fork", status: "resolved_portable" },
2646 ]);
2647 expect(harness.startTurn.mock.calls[0]?.[0].message.text).toContain("PARENT_HISTORY");
2648 expect(harness.projection().providerThreads[0]?.handoffIds).toEqual([
2649 "side-chat-fork-context",
2650 ]);
2651 expect(harness.settledRuns).toEqual([]);
2652 expect(
2653 harness.prepareProviderHandoff.mock.calls[0]?.[0].items.map((item) => item.id),
2654 ).toEqual([items[0]!.id]);
2655 }),
2656 );
2657
2658 effectIt.effect.each([
2659 { runStatus: "failed", turnStatus: "completed" },
2660 { runStatus: "completed", turnStatus: "completed" },
2661 { runStatus: "failed", turnStatus: "running", differentThread: true },
2662 { runStatus: "failed", turnStatus: "running", liveSession: false },
2663 { runStatus: "failed", turnStatus: "interrupted", interruptFailure: true },
2664 ] as const)("continues without interrupting when recovery does not apply: %j", (previousTurn) =>
2665 Effect.gen(function* () {
2666 const harness = makeLocalCommandHarness({ text: "Continue", previousTurn });
2667
2668 yield* harness.start;
2669
2670 expect(harness.interruptTurn).not.toHaveBeenCalled();
2671 expect(harness.projection().runs.at(-1)?.status).toBe("running");
2672 }),
2673 );
2674
2675 effectIt.effect("leaves delegated children to their own runs when the parent starts a turn", () =>
2676 Effect.gen(function* () {
2677 const harness = makeLocalCommandHarness({
2678 text: "Continue",
2679 previousTurn: { runStatus: "completed", turnStatus: "completed" },
2680 });
2681 const projection = harness.projection();
2682 const completedChild = (origin: "provider_native" | "app_owned") => ({
2683 id: NodeId.make(`node:${origin}`),
2684 threadId: projection.thread.id,
2685 runId: projection.runs[0]!.id,
2686 parentNodeId: NodeId.make(`node:${origin}:parent`),
2687 origin,
2688 createdBy: "agent" as const,
2689 driver: ProviderDriverKind.make("codex"),
2690 providerInstanceId: ProviderInstanceId.make("codex"),
2691 providerThreadId: ProviderThreadId.make(`provider-thread:${origin}`),
2692 childThreadId: ThreadId.make(`thread:${origin}`),
2693 nativeTaskRef: null,
2694 prompt: "work",
2695 title: null,
2696 model: null,
2697 status: "completed" as const,
2698 result: "done",
2699 startedAt: null,
2700 completedAt: null,
2701 updatedAt: projection.updatedAt,
2702 });
2703 harness.setProjection({
2704 ...projection,
2705 subagents: [completedChild("provider_native"), completedChild("app_owned")],
2706 });
2707
2708 yield* harness.start;
2709
2710 // A delegated child thread has its own run on the shared session; a parent run that also
2711 // owns it ingests every one of the child's later items a second time.
2712 expect(harness.startRootRun.mock.calls[0]?.[0]).toMatchObject({
2713 relatedThreadIds: [ThreadId.make("thread:provider_native")],
2714 relatedProviderThreadIds: [ProviderThreadId.make("provider-thread:provider_native")],
2715 });
2716 }),
2717 );
2718
2719 effectIt.effect("does not start another turn when abandoned-turn cleanup fails", () =>
2720 Effect.gen(function* () {
2721 const harness = makeLocalCommandHarness({
2722 text: "Continue",
2723 previousTurn: { runStatus: "failed", turnStatus: "running", interruptFailure: true },
2724 });
2725
2726 yield* harness.start;
2727
2728 expect(harness.interruptTurn).toHaveBeenCalledOnce();
2729 expect(harness.startRootRun).not.toHaveBeenCalled();
2730 expect(harness.projection().runs.at(-1)?.status).toBe("failed");
2731 }),
2732 );
2733
2734 effectIt.effect("retries abandoned-turn cleanup without failing the continuation early", () =>
2735 Effect.gen(function* () {
2736 const harness = makeLocalCommandHarness({
2737 text: "Continue",
2738 previousTurn: { runStatus: "failed", turnStatus: "running", interruptFailure: true },
2739 });
2740
2741 const error = yield* harness.startWithRetry.pipe(Effect.flip);
2742
2743 expect(error._tag).toBe("ProviderTurnStartError");
2744 expect(harness.projection().runs.at(-1)?.status).toBe("starting");
2745 expect(harness.open).not.toHaveBeenCalled();
2746 }),
2747 );
2748
2749 effectIt.effect("does not start the continuation if recording recovery fails", () =>
2750 Effect.gen(function* () {
2751 const harness = makeLocalCommandHarness({
2752 text: "Continue",
2753 previousTurn: { runStatus: "failed", turnStatus: "running" },
2754 writeFailure: new Error("database unavailable"),
2755 });
2756
2757 const error = yield* harness.start.pipe(Effect.flip);
2758
2759 expect(error._tag).toBe("ProviderTurnStartError");
2760 expect(harness.projection().runs.at(-1)?.status).toBe("starting");
2761 expect(harness.projection().providerTurns[0]?.status).toBe("running");
2762 expect(harness.open).not.toHaveBeenCalled();
2763 }),
2764 );
2765
2766 effectIt.effect("does not resume a continuation stopped while recovery was in flight", () =>
2767 Effect.gen(function* () {
2768 const harness = makeLocalCommandHarness({
2769 text: "Continue",
2770 previousTurn: { runStatus: "failed", turnStatus: "running", interruptCurrentRun: true },
2771 });
2772
2773 yield* harness.start;
2774
2775 expect(harness.projection().runs.at(-1)?.status).toBe("interrupted");
2776 expect(harness.open).not.toHaveBeenCalled();
2777 }),
2778 );
2779
2780 effectIt.effect.each(["cleanup-first", "startup-first"])(
2781 "coordinates provider startup with %s",
2782 (order) =>
2783 Effect.gen(function* () {
2784 const fileSystem = yield* FileSystem.FileSystem;
2785 const path = yield* Path.Path;
2786 const workspaceRoot = yield* fileSystem.makeTempDirectoryScoped({ prefix: "t3-start-" });
2787 const worktreePath = path.join(workspaceRoot, "project", "worktree");
2788 yield* fileSystem.makeDirectory(worktreePath, { recursive: true });
2789 const release = yield* Deferred.make<void>();
2790 const openEntered = yield* Deferred.make<void>();
2791 const harness = makeLocalCommandHarness({
2792 text: "Continue",
2793 worktreePath,
2794 worktreesDir: workspaceRoot,
2795 fileSystem,
2796 failReadsAfterRunning: true,
2797 });
2798 harness.createWorktree.mockImplementation(() =>
2799 fileSystem
2800 .makeDirectory(worktreePath, { recursive: true })
2801 .pipe(Effect.as({} as never), Effect.orDie),
2802 );
2803 const open = harness.open.getMockImplementation()!;
2804 harness.open.mockImplementation(() =>
2805 Effect.gen(function* () {
2806 expect(yield* fileSystem.exists(worktreePath).pipe(Effect.orDie)).toBe(true);
2807 yield* Deferred.succeed(openEntered, undefined);
2808 if (order === "startup-first") yield* Deferred.await(release);
2809 return yield* open();
2810 }),
2811 );
2812 let cleanupEntered = false;
2813 const cleanupEffect = withWorkspaceLease(
2814 worktreePath,
2815 Effect.gen(function* () {
2816 cleanupEntered = true;
2817 if (order === "cleanup-first") yield* Deferred.await(release);
2818 else {
2819 expect(harness.projection().providerSessions).toHaveLength(1);
2820 expect(harness.projection().runs.at(-1)?.status).toBe("running");
2821 }
2822 yield* fileSystem.remove(worktreePath, { recursive: true, force: true });
2823 }),
2824 );
2825 const cleanup =
2826 order === "cleanup-first"
2827 ? yield* cleanupEffect.pipe(Effect.forkChild({ startImmediately: true }))
2828 : undefined;
2829 const startup = yield* harness.start.pipe(Effect.forkChild({ startImmediately: true }));
2830 if (order === "cleanup-first") expect(harness.open).not.toHaveBeenCalled();
2831 if (order === "startup-first") yield* Deferred.await(openEntered);
2832 const laterCleanup =
2833 order === "startup-first"
2834 ? yield* cleanupEffect.pipe(Effect.forkChild({ startImmediately: true }))
2835 : undefined;
2836 if (order === "startup-first") expect(cleanupEntered).toBe(false);
2837 yield* Deferred.succeed(release, undefined);
2838 if (cleanup !== undefined) yield* Fiber.join(cleanup);
2839 yield* Fiber.join(startup);
2840 if (laterCleanup !== undefined) yield* Fiber.join(laterCleanup);
2841 if (order === "cleanup-first") {
2842 expect(harness.createWorktree).toHaveBeenCalledWith({
2843 cwd: "/tmp/native-account-project",
2844 refName: "feature/restore",
2845 path: worktreePath,
2846 });
2847 }
2848 expect(harness.resolveRuntimePolicy).toHaveBeenCalledWith(
2849 expect.objectContaining({
2850 thread: expect.objectContaining({ worktreePath }),
2851 }),
2852 );
2853 expect(harness.open).toHaveBeenCalledWith(
2854 expect.objectContaining({
2855 runtimePolicy: expect.objectContaining({ cwd: worktreePath }),
2856 }),
2857 );
2858 expect(harness.startRootRun).toHaveBeenCalledWith(
2859 expect.objectContaining({
2860 appThread: expect.objectContaining({ worktreePath }),
2861 runtimePolicy: expect.objectContaining({ cwd: worktreePath }),
2862 }),
2863 );
2864 }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
2865 );
2866
2867 // regression: #12341 a turn start overlapping a checkpoint restore wrote into the checkout
2868 // The filesystem is synchronous so the turn start either reaches the provider or parks on the
2869 // lease before the restore is released; realPath models the symlink alias and missing worktrees.
2870 effectIt.effect.each([
2871 {
2872 overlap: "a project-root thread during a restore of the project checkout",
2873 restorePath: "/tmp/native-account-project",
2874 worktreePath: undefined,
2875 realPaths: {},
2876 missing: [],
2877 },
2878 {
2879 overlap: "a symlink alias of the restoring checkout",
2880 restorePath: "/tmp/restore-overlap/link",
2881 worktreePath: "/tmp/restore-overlap/checkout",
2882 realPaths: { "/tmp/restore-overlap/link": "/tmp/restore-overlap/checkout" },
2883 missing: [],
2884 },
2885 {
2886 overlap: "a directory inside the restoring checkout",
2887 restorePath: "/tmp/restore-overlap/checkout",
2888 worktreePath: "/tmp/restore-overlap/checkout/packages/app",
2889 realPaths: {},
2890 missing: [],
2891 },
2892 {
2893 overlap: "a not-yet-recreated worktree under a symlink alias of the restoring one",
2894 restorePath: "/tmp/restore-overlap/link/missing-child",
2895 worktreePath: "/tmp/restore-overlap/checkout/missing-child",
2896 realPaths: { "/tmp/restore-overlap/link": "/tmp/restore-overlap/checkout" },
2897 missing: [
2898 "/tmp/restore-overlap/link/missing-child",
2899 "/tmp/restore-overlap/checkout/missing-child",
2900 ],
2901 },
2902 ] as const)(
2903 "does not start provider work in $overlap until the restore finishes",
2904 ({ restorePath, worktreePath, realPaths, missing }) =>
2905 Effect.gen(function* () {
2906 const order: Array<string> = [];
2907 const releaseRestore = yield* Deferred.make<void>();
2908 const restoreEntered = yield* Deferred.make<void>();
2909 const fileSystem = FileSystem.makeNoop({
2910 exists: () => Effect.succeed(true),
2911 realPath: (candidate) =>
2912 (missing as ReadonlyArray<string>).includes(candidate)
2913 ? Effect.fail(
2914 PlatformError.systemError({
2915 _tag: "NotFound",
2916 module: "FileSystem",
2917 method: "realPath",
2918 pathOrDescriptor: candidate,
2919 }),
2920 )
2921 : Effect.succeed((realPaths as Record<string, string>)[candidate] ?? candidate),
2922 });
2923 const harness = makeLocalCommandHarness({
2924 text: "Continue",
2925 ...(worktreePath === undefined ? {} : { worktreePath }),
2926 fileSystem,
2927 failReadsAfterRunning: true,
2928 });
2929 const open = harness.open.getMockImplementation()!;
2930 harness.open.mockImplementation(() =>
2931 Effect.sync(() => order.push("provider session opened")).pipe(Effect.andThen(open())),
2932 );
2933 // The reservation CheckpointRollbackServiceV2.execute holds around its
2934 // isolation check, provider rollback, and git restore of `restorePath`.
2935 const restore = yield* withCheckpointRestoreReservation(
2936 fileSystem,
2937 yield* Path.Path,
2938 restorePath,
2939 Deferred.succeed(restoreEntered, undefined).pipe(
2940 Effect.andThen(Deferred.await(releaseRestore)),
2941 Effect.andThen(Effect.sync(() => order.push("checkpoint files restored"))),
2942 ),
2943 ).pipe(Effect.forkChild({ startImmediately: true }));
2944 yield* Deferred.await(restoreEntered);
2945 const startup = yield* harness.start.pipe(Effect.forkChild({ startImmediately: true }));
2946 yield* Deferred.succeed(releaseRestore, undefined);
2947 yield* Fiber.join(restore);
2948 yield* Fiber.join(startup);
2949 expect(order).toEqual(["checkpoint files restored", "provider session opened"]);
2950 }).pipe(Effect.provide(Path.layer)),
2951 );
2952
2953 effectIt.effect("reads current startup state after waiting for the workspace lease", () =>
2954 Effect.gen(function* () {
2955 const worktreePath = "/tmp/provider-start-fresh-state";
2956 const release = yield* Deferred.make<void>();
2957 const readEntered = yield* Deferred.make<void>();
2958 const harness = makeLocalCommandHarness({
2959 text: "Continue",
2960 worktreePath,
2961 failReadsAfterRunning: true,
2962 });
2963 harness.getThreadRecords.mockImplementationOnce(() =>
2964 Deferred.succeed(readEntered, undefined).pipe(Effect.as(harness.projection())),
2965 );
2966 const holder = yield* withWorkspaceLease(worktreePath, Deferred.await(release)).pipe(
2967 Effect.forkChild({ startImmediately: true }),
2968 );
2969 const startup = yield* harness.start.pipe(Effect.forkChild({ startImmediately: true }));
2970 yield* Deferred.await(readEntered);
2971 expect(harness.getTurnStartContext).not.toHaveBeenCalled();
2972 const projection = harness.projection();
2973 harness.setProjection({
2974 ...projection,
2975 runs: projection.runs.map((run) => ({ ...run, status: "running" })),
2976 });
2977 yield* Deferred.succeed(release, undefined);
2978 yield* Fiber.join(holder);
2979 yield* Fiber.join(startup);
2980 expect(harness.open).not.toHaveBeenCalled();
2981 expect(harness.startRootRun).not.toHaveBeenCalled();
2982 }),
2983 );
2984
2985 // regression: #15784 superseded provider startups kept running
2986 effectIt.effect.each([
2987 { boundary: "policy resolution", loss: "replaced" },
2988 { boundary: "session opening", loss: "interrupted" },
2989 ])("does not continue startup when $loss during $boundary", ({ boundary, loss }) =>
2990 Effect.gen(function* () {
2991 const entered = yield* Deferred.make<void>();
2992 const release = yield* Deferred.make<void>();
2993 const gate = Deferred.succeed(entered, undefined).pipe(Effect.andThen(Deferred.await(release)));
2994 const harness = makeLocalCommandHarness({ text: "Continue" });
2995 const projection = harness.projection();
2996 const providerThread = projection.providerThreads[0]!;
2997 const ensureThread = vi.fn(() => Effect.succeed(providerThread));
2998 const resumeThread = vi.fn(() => Effect.succeed(providerThread));
2999 const forkThread = vi.fn(() => Effect.succeed(providerThread));
3000 harness.open.mockReturnValue(
3001 (boundary === "session opening" ? gate : Effect.void).pipe(
3002 Effect.as({
3003 driver: providerThread.driver,
3004 ensureThread,
3005 resumeThread,
3006 forkThread,
3007 } as never),
3008 ),
3009 );
3010 if (boundary === "policy resolution") {
3011 harness.resolveRuntimePolicy.mockReturnValue(
3012 gate.pipe(Effect.as({ cwd: "/tmp/native-account-command" } as never)),
3013 );
3014 }
3015
3016 const startup = yield* harness.start.pipe(Effect.forkChild({ startImmediately: true }));
3017 yield* Deferred.await(entered);
3018 const current = harness.projection();
3019 const replacementId = RunAttemptId.make("replacement-attempt");
3020 const updated = {
3021 ...current,
3022 runs: current.runs.map((candidate) =>
3023 loss === "replaced"
3024 ? { ...candidate, activeAttemptId: replacementId }
3025 : { ...candidate, status: "interrupted" as const, completedAt: current.updatedAt },
3026 ),
3027 attempts:
3028 loss === "replaced"
3029 ? [...current.attempts, { ...current.attempts[0]!, id: replacementId, attemptOrdinal: 2 }]
3030 : current.attempts,
3031 };
3032 harness.setProjection(updated);
3033 yield* Deferred.succeed(release, undefined);
3034 yield* Fiber.join(startup);
3035
3036 expect(harness.open).toHaveBeenCalledTimes(boundary === "session opening" ? 1 : 0);
3037 expect(ensureThread).not.toHaveBeenCalled();
3038 expect(resumeThread).not.toHaveBeenCalled();
3039 expect(forkThread).not.toHaveBeenCalled();
3040 expect(harness.startRootRun).not.toHaveBeenCalled();
3041 expect(harness.writeIfRunCurrent).not.toHaveBeenCalled();
3042 expect(harness.events).toEqual([]);
3043 expect(harness.projection()).toEqual(updated);
3044 }),
3045 );
3046
3047 effectIt.effect("starts another provider in the same checkout while a turn is pending", () =>
3048 Effect.gen(function* () {
3049 const worktreePath = "/tmp/provider-start-shared-checkout";
3050 const turnEntered = yield* Deferred.make<void>();
3051 const releaseTurn = yield* Deferred.make<void>();
3052 const first = makeLocalCommandHarness({
3053 text: "Continue",
3054 worktreePath,
3055 failReadsAfterRunning: true,
3056 });
3057 const second = makeLocalCommandHarness({
3058 text: "Continue",
3059 worktreePath,
3060 failReadsAfterRunning: true,
3061 });
3062 first.startRootRun.mockImplementation(() =>
3063 Deferred.succeed(turnEntered, undefined).pipe(Effect.andThen(Deferred.await(releaseTurn))),
3064 );
3065 const pending = yield* first.start.pipe(Effect.forkChild({ startImmediately: true }));
3066 yield* Deferred.await(turnEntered);
3067 yield* second.start;
3068 expect(second.startRootRun).toHaveBeenCalledOnce();
3069 expect(second.projection().runs.at(-1)?.status).toBe("running");
3070 yield* Deferred.succeed(releaseTurn, undefined);
3071 yield* Fiber.join(pending);
3072 }),
3073 );
3074
3075 effectIt.effect("fails a starting run when its recovery state cannot be read", () =>
3076 Effect.gen(function* () {
3077 const harness = makeLocalCommandHarness({
3078 text: "Continue",
3079 recoveryReadFailure: "stored turn item no longer decodes",
3080 });
3081
3082 yield* harness.start.pipe(Effect.ignore);
3083
3084 expect(harness.open).not.toHaveBeenCalled();
3085 expect(harness.startRootRun).not.toHaveBeenCalled();
3086 const projection = harness.projection();
3087 expect(projection.runs.at(-1)).toMatchObject({ status: "failed", startedAt: null });
3088 expect(projection.attempts[0]).toMatchObject({ status: "failed", startedAt: null });
3089 expect(projection.nodes[0]).toMatchObject({ status: "failed", startedAt: null });
3090 expect(projection.turnItems).toMatchObject([{ type: "error", status: "failed" }]);
3091 }),
3092 );
3093
3094 effectIt.effect("terminalizes a starting run when its provider session cannot open", () =>
3095 Effect.gen(function* () {
3096 const harness = makeLocalCommandHarness({
3097 text: "Continue",
3098 openFailure: new Error("DESCRIPTION is not valid ACP JSON"),
3099 });
3100
3101 yield* harness.start;
3102
3103 expect(harness.open).toHaveBeenCalledOnce();
3104 expect(harness.startRootRun).not.toHaveBeenCalled();
3105 expect(harness.writeIfRunCurrent).toHaveBeenCalledWith(
3106 expect.objectContaining({
3107 activeAttemptId: harness.attemptId,
3108 expectedStatus: "starting",
3109 }),
3110 );
3111 const projection = harness.projection();
3112 expect(projection.runs.at(-1)).toMatchObject({ status: "failed", startedAt: null });
3113 expect(harness.settledRuns).toEqual(["failed"]);
3114 expect(projection.attempts[0]).toMatchObject({ status: "failed", startedAt: null });
3115 expect(projection.nodes[0]).toMatchObject({ status: "failed", startedAt: null });
3116 expect(projection.turnItems).toMatchObject([
3117 {
3118 type: "error",
3119 status: "failed",
3120 failure: {
3121 class: "provider_error",
3122 message: "DESCRIPTION is not valid ACP JSON",
3123 },
3124 },
3125 ]);
3126 }),
3127 );
3128
3129 effectIt.effect("leaves the run starting when a session-open failure will be retried", () =>
3130 Effect.gen(function* () {
3131 const harness = makeLocalCommandHarness({
3132 text: "Continue",
3133 openFailure: new Error("provider session rejected"),
3134 });
3135
3136 const error = yield* harness.startWithRetry.pipe(Effect.flip);
3137
3138 expect(error._tag).toBe("ProviderTurnStartError");
3139 expect(harness.writeIfRunCurrent).not.toHaveBeenCalled();
3140 expect(harness.projection().runs.at(-1)?.status).toBe("starting");
3141 }),
3142 );
3143
3144 effectIt.effect("fails the run without a retry when the provider needs sign-in", () =>
3145 Effect.gen(function* () {
3146 const harness = makeLocalCommandHarness({
3147 text: "Continue",
3148 openFailure: new ProviderAdapter.ProviderAdapterOpenSessionError({
3149 driver: ProviderDriverKind.make("antigravity"),
3150 providerSessionId: ProviderSessionId.make("new-provider-session"),
3151 requiresSignIn: true,
3152 }),
3153 });
3154
3155 yield* harness.startWithRetry;
3156
3157 expect(harness.open).toHaveBeenCalledOnce();
3158 expect(harness.projection().runs.at(-1)?.status).toBe("failed");
3159 expect(harness.projection().turnItems).toMatchObject([
3160 {
3161 type: "error",
3162 status: "failed",
3163 failure: {
3164 class: "provider_error",
3165 message: "Failed to open antigravity provider session new-provider-session.",
3166 },
3167 },
3168 ]);
3169 }),
3170 );
3171
3172 effectIt.effect("does not terminalize a provider-session open interruption", () =>
3173 Effect.gen(function* () {
3174 const harness = makeLocalCommandHarness({ text: "Continue", interruptOpen: true });
3175
3176 const exit = yield* Effect.exit(harness.start);
3177
3178 expect(exit._tag).toBe("Failure");
3179 if (exit._tag === "Failure") {
3180 expect(Cause.hasInterruptsOnly(exit.cause)).toBe(true);
3181 }
3182 expect(harness.writeIfRunCurrent).not.toHaveBeenCalled();
3183 expect(harness.startRootRun).not.toHaveBeenCalled();
3184 expect(harness.projection().runs.at(-1)?.status).toBe("starting");
3185 expect(harness.events).toEqual([]);
3186 }),
3187 );
3188
3189 effectIt.effect("does not overwrite a run interrupted while its provider session opens", () =>
3190 Effect.gen(function* () {
3191 const harness = makeLocalCommandHarness({
3192 text: "Continue",
3193 openFailure: new Error("provider session rejected"),
3194 interruptRunBeforeOpenFailure: true,
3195 });
3196
3197 yield* harness.start;
3198
3199 expect(harness.writeIfRunCurrent).toHaveBeenCalledOnce();
3200 expect(harness.startRootRun).not.toHaveBeenCalled();
3201 const projection = harness.projection();
3202 expect(projection.runs.at(-1)?.status).toBe("interrupted");
3203 expect(harness.settledRuns).toEqual([]);
3204 expect(projection.attempts[0]?.status).toBe("pending");
3205 expect(projection.nodes[0]?.status).toBe("pending");
3206 expect(projection.turnItems).toEqual([]);
3207 expect(harness.events).toEqual([]);
3208 }),
3209 );
3210
3211 effectIt.effect("checks the run's status without loading the thread's history", () =>
3212 Effect.gen(function* () {
3213 const harness = makeLocalCommandHarness({
3214 text: "Continue",
3215 previousTurn: { runStatus: "completed", turnStatus: "completed" },
3216 });
3217
3218 yield* harness.start;
3219
3220 const controls = harness.startRootRun.mock.calls[0]?.[0];
3221 expect(controls).toBeDefined();
3222 if (controls === undefined) return;
3223 expect(yield* controls.shouldStartProviderTurn!()).toBe(true);
3224 expect(yield* controls.shouldFinalizeRun!()).toBe(true);
3225 // Each check needs one run row; the recovery projection reads every run and open turn item.
3226 // The one read left is the start's inherited background items snapshot, not a status check.
3227 expect(
3228 harness.getRuntimeRecoveryProjection.mock.calls.length,
3229 "recovery projection reads beyond the inherited background items snapshot",
3230 ).toBe(1);
3231 }),
3232 );
3233
3234 effectIt.effect("fails the run instead of replacing a native thread another client holds", () =>
3235 Effect.gen(function* () {
3236 const detail = "This conversation is open in another Codex client.";
3237 const harness = makeLocalCommandHarness({
3238 text: "Continue",
3239 resumeFailure: new ProviderAdapter.ProviderAdapterThreadInUseError({
3240 driver: ProviderDriverKind.make("codex"),
3241 providerThreadId: ProviderThreadId.make("new-provider-thread"),
3242 detail,
3243 cause: new Error("thread native-resume-thread already has an active writer"),
3244 }),
3245 });
3246
3247 const exit = yield* Effect.exit(harness.start);
3248
3249 expect(harness.fallbackEnsureThread).not.toHaveBeenCalled();
3250 expect(exit._tag).toBe("Success");
3251 expect(harness.startRootRun).not.toHaveBeenCalled();
3252 const projection = harness.projection();
3253 expect(projection.providerThreads.at(-1)?.nativeThreadRef?.nativeId).toBe(
3254 "native-resume-thread",
3255 );
3256 expect(projection.contextHandoffs).toEqual([]);
3257 expect(projection.runs.at(-1)).toMatchObject({ status: "failed", startedAt: null });
3258 expect(projection.turnItems).toMatchObject([
3259 {
3260 type: "error",
3261 title: "Provider turn failed to start",
3262 failure: { class: "provider_error", message: detail },
3263 },
3264 ]);
3265 }),
3266 );
3267
3268 effectIt.effect("does not mistake a failed state read for a superseded run", () =>
3269 Effect.gen(function* () {
3270 const harness = makeLocalCommandHarness({ text: "Continue", failReadsAfterRunning: true });
3271
3272 yield* harness.start;
3273
3274 expect(harness.projection().runs.at(-1)?.status).toBe("running");
3275 const controls = harness.startRootRun.mock.calls[0]?.[0];
3276 expect(controls).toBeDefined();
3277 if (controls === undefined) return;
3278 const startCheck = yield* Effect.flip(controls.shouldStartProviderTurn!());
3279 const finalizeCheck = yield* Effect.flip(controls.shouldFinalizeRun!());
3280 expect(startCheck._tag).toBe("ProjectionStoreReadError");
3281 expect(finalizeCheck._tag).toBe("ProjectionStoreReadError");
3282 }),
3283 );
3284
3285 effectIt.effect.each(["not_loaded", "error"] as const)(
3286 "signs out the existing native provider before opening the newly selected provider with status %s",
3287 (providerThreadStatus) =>
3288 Effect.gen(function* () {
3289 const harness = makeLocalCommandHarness({ text: "/logout", previousNativeSession: true });
3290 const initial = harness.projection();
3291 harness.setProjection({
3292 ...initial,
3293 providerThreads: initial.providerThreads.map((thread) =>
3294 thread.id === "new-provider-thread"
3295 ? { ...thread, status: providerThreadStatus }
3296 : thread,
3297 ),
3298 });
3299
3300 yield* harness.start;
3301 yield* harness.start;
3302
3303 expect(harness.tryHandlePromptCommand).toHaveBeenCalledExactlyOnceWith({
3304 instanceId: harness.oldInstanceId,
3305 text: "/logout",
3306 hasAttachments: false,
3307 sentByPerson: true,
3308 });
3309 expect(harness.open).not.toHaveBeenCalled();
3310 expect(harness.startRootRun).not.toHaveBeenCalled();
3311 const projection = harness.projection();
3312 expect(projection.runs.at(-1)?.status).toBe("completed");
3313 expect(projection.attempts[0]?.status).toBe("completed");
3314 expect(projection.nodes[0]?.status).toBe("completed");
3315 expect(projection.turnItems).toMatchObject([
3316 {
3317 type: "command_execution",
3318 title: "Provider signed out",
3319 output: "Provider signed out",
3320 status: "completed",
3321 },
3322 ]);
3323 expect(projection.providerTurns).toEqual([]);
3324 expect(projection.checkpoints).toEqual([]);
3325 expect(
3326 projection.providerThreads.find((thread) => thread.id === "new-provider-thread")?.status,
3327 ).toBe(providerThreadStatus);
3328 }),
3329 );
3330
3331 effectIt.effect.each([
3332 { name: "an agent", createdBy: "agent" as const },
3333 // A scheduled delivery keeps its task author's createdBy.
3334 { name: "a scheduled task", scheduledTaskId: ScheduledTaskId.make("task") },
3335 ])("does not let $name sign a provider out", ({ name: _name, ...sender }) =>
3336 Effect.gen(function* () {
3337 const harness = makeLocalCommandHarness({
3338 text: "/logout",
3339 ...sender,
3340 previousNativeSession: true,
3341 });
3342
3343 yield* harness.start;
3344
3345 expect(harness.tryHandlePromptCommand).toHaveBeenCalledExactlyOnceWith(
3346 expect.objectContaining({ sentByPerson: false }),
3347 );
3348 }),
3349 );
3350
3351 for (const previousMessages of [[], ["/compact", " /COMPACT "]]) {
3352 effectIt.effect(
3353 `rejects compaction without conversation context after ${previousMessages.length} prior compactions`,
3354 () =>
3355 Effect.gen(function* () {
3356 const harness = makeLocalCommandHarness({ text: "/compact", previousMessages });
3357
3358 yield* harness.start;
3359
3360 expect(harness.open).not.toHaveBeenCalled();
3361 expect(harness.tryHandlePromptCommand).not.toHaveBeenCalled();
3362 expect(harness.startRootRun).not.toHaveBeenCalled();
3363 expect(harness.projection().runs.at(-1)?.status).toBe("failed");
3364 expect(harness.projection().turnItems).toMatchObject([
3365 {
3366 type: "error",
3367 failure: {
3368 class: "validation_error",
3369 message: "Start a conversation before compacting this thread.",
3370 },
3371 },
3372 ]);
3373 }),
3374 );
3375 }
3376
3377 effectIt.effect("rejects citation expansion before opening a provider session", () =>
3378 Effect.gen(function* () {
3379 const marker = serializeAssistantCitation({
3380 version: 1,
3381 environmentId: EnvironmentId.make("env"),
3382 threadId: ThreadId.make("thread"),
3383 messageId: MessageId.make("assistant"),
3384 text: "\u0001".repeat(8_000),
3385 start: 0,
3386 end: 8_000,
3387 prefix: "",
3388 suffix: "",
3389 });
3390 const text = "x".repeat(80_000) + marker;
3391 expect(text.length).toBeLessThan(PROVIDER_SEND_TURN_MAX_INPUT_CHARS);
3392 expect(expandAssistantCitationsForProvider(text).length).toBeGreaterThan(
3393 PROVIDER_SEND_TURN_MAX_INPUT_CHARS,
3394 );
3395 const harness = makeLocalCommandHarness({ text });
3396 yield* harness.start;
3397 expect(harness.open).not.toHaveBeenCalled();
3398 expect(harness.startTurn).not.toHaveBeenCalled();
3399 expect(harness.projection().runs.at(-1)?.status).toBe("failed");
3400 expect(harness.projection().turnItems).toMatchObject([
3401 {
3402 type: "error",
3403 failure: { class: "validation_error", message: expect.stringContaining("120,000") },
3404 },
3405 ]);
3406 expect(harness.projection().messages[0]?.text).toBe(text);
3407 }),
3408 );