The best way to control Claude Code, Codex and more from any computer or phone

main
ts 2,553 lines 94.8 KB
Raw
1 import { assert, it } from "@effect/vitest";
2 import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
3 import * as NodeServices from "@effect/platform-node/NodeServices";
4 import {
5 CheckpointId,
6 CheckpointRef,
7 CheckpointScopeId,
8 MessageId,
9 GitCommandError,
10 type ModelSelection,
11 NodeId,
12 type OrchestrationV2AppThread,
13 type OrchestrationV2DomainEvent,
14 type OrchestrationV2ExecutionNode,
15 type OrchestrationV2RuntimeRequest,
16 type OrchestrationV2ProviderThread,
17 type OrchestrationV2Run,
18 type OrchestrationV2TurnItem,
19 ProviderDriverKind,
20 ProviderInstanceId,
21 ProviderThreadId,
22 ProviderTurnId,
23 RunAttemptId,
24 RunId,
25 RuntimeRequestId,
26 TurnItemId,
27 } from "@t3tools/contracts";
28 import * as Cause from "effect/Cause";
29 import * as DateTime from "effect/DateTime";
30 import * as Deferred from "effect/Deferred";
31 import * as Fiber from "effect/Fiber";
32 import * as Effect from "effect/Effect";
33 import * as Exit from "effect/Exit";
34 import * as FileSystem from "effect/FileSystem";
35 import * as Layer from "effect/Layer";
36 import * as Schema from "effect/Schema";
37 import * as SqlClient from "effect/sql/SqlClient";
38 import * as SqlError from "effect/sql/SqlError";
39 import * as Stream from "effect/Stream";
40 import * as Tracer from "effect/Tracer";
41
42 import { toolOutputImages } from "@t3tools/shared/toolOutput";
43 import * as SqlitePersistence from "../persistence/Sqlite.ts";
44 import * as ServerConfig from "../config.ts";
45 import * as GitVcsDriver from "../vcs/GitVcsDriver.ts";
46 import * as EventSink from "./EventSink.ts";
47 import * as EventStore from "./EventStore.ts";
48 import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
49 import * as ProjectionStore from "./ProjectionStore.ts";
50 import * as ProviderEventIngestor from "./ProviderEventIngestor.ts";
51 import * as ProviderUsageAnalytics from "../telemetry/ProviderUsageAnalytics.ts";
52 import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
53 import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.ts";
54 import { UsageLimitResetFallback } from "../usage/UsageLimitSources.ts";
55 import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
56 import type { ProviderInstance } from "@t3tools/provider-core/server/driver";
57 import type { ServerProvider } from "@t3tools/contracts";
58 import { makeProviderFailure } from "@t3tools/provider-core/server/failure";
59 import {
60 makeProviderEventRoutingState,
61 type ProviderEventRouteIdentity,
62 routeProviderEvent,
63 selectInheritedBackgroundTurnItems,
64 } from "./RunExecutionService.ts";
65
66 const layerTestDatabase = SqlitePersistence.layerMemory;
67 const layerTestStores = Layer.merge(EventStore.layer, ProjectionStore.layer).pipe(
68 Layer.provide(layerTestDatabase),
69 );
70
71 const layerTestEventSink = EventSink.layer.pipe(
72 Layer.provide(Layer.mergeAll(layerTestStores, layerTestDatabase)),
73 );
74
75 const layerGit = GitVcsDriver.layer.pipe(
76 Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "ingestor-git-" })),
77 Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)),
78 );
79
80 const layerEmptyInstances = Layer.mock(ProviderInstanceRegistry.ProviderInstanceRegistry)({
81 getInstance: () => Effect.undefined,
82 });
83 const makeLayerTest = <E>(
84 ingestorEventSink: Layer.Layer<EventSink.EventSinkV2, E>,
85 instances = layerEmptyInstances,
86 ) =>
87 Layer.mergeAll(
88 instances,
89 layerTestDatabase,
90 layerTestStores,
91 layerTestEventSink,
92 IdAllocator.layer,
93 ThreadCommandExecutor.layer,
94 layerGit,
95 ProviderEventIngestor.layer.pipe(
96 Layer.provide(
97 Layer.mergeAll(
98 instances,
99 layerTestStores,
100 ingestorEventSink,
101 IdAllocator.layer,
102 ThreadCommandExecutor.layer,
103 layerGit,
104 ),
105 ),
106 ),
107 );
108 const layerTest = makeLayerTest(layerTestEventSink);
109
110 // EventSink reports a lock that outlasted busy_timeout at BEGIN IMMEDIATE this way.
111 const lockTimeout = new EventSink.EventSinkWriteError({
112 eventCount: 1,
113 cause: new SqlError.SqlError({ reason: new SqlError.LockTimeoutError({ cause: null }) }),
114 });
115 const constraintFailure = new EventSink.EventSinkWriteError({
116 eventCount: 1,
117 cause: new SqlError.SqlError({ reason: new SqlError.ConstraintError({ cause: null }) }),
118 });
119
120 /** The ingestor's next sink writes of `events` fail with `errors`, in order; the rest go through. */
121 const layerFailingWrites = (
122 errors: ReadonlyArray<EventSink.EventSinkWriteError>,
123 matches: (events: ReadonlyArray<OrchestrationV2DomainEvent>) => boolean = () => true,
124 ) =>
125 makeLayerTest(
126 Layer.effect(
127 EventSink.EventSinkV2,
128 Effect.gen(function* () {
129 const eventSink = yield* EventSink.EventSinkV2;
130 const pending = [...errors];
131 const failNext = <A, R>(
132 events: ReadonlyArray<OrchestrationV2DomainEvent>,
133 write: Effect.Effect<A, EventSink.EventSinkV2Error, R>,
134 ) => Effect.suspend(() => (matches(events) ? (pending.shift() ?? write) : write));
135 return EventSink.EventSinkV2.of({
136 ...eventSink,
137 write: (input) => failNext(input.events, eventSink.write(input)),
138 writeIfRunCurrent: (input) => failNext(input.events, eventSink.writeIfRunCurrent(input)),
139 writeIfProviderThreadOwner: (input) =>
140 failNext(input.events, eventSink.writeIfProviderThreadOwner(input)),
141 });
142 }),
143 ).pipe(Layer.provide(layerTestEventSink)),
144 );
145
146 const makeSharedWorktree = Effect.gen(function* () {
147 const fs = yield* FileSystem.FileSystem;
148 const git = yield* GitVcsDriver.GitVcsDriver;
149 const directory = yield* fs.makeTempDirectoryScoped({ prefix: "ingestor-worktree-" });
150 const repo = `${directory}/repo`;
151 const worktreePath = `${directory}/shared`;
152 yield* fs.makeDirectory(repo);
153 const command = (cwd: string, args: string[]) =>
154 git.execute({ operation: "ingestor-test", cwd, args });
155 yield* command(repo, ["init", "-b", "main"]);
156 yield* command(repo, [
157 "-c",
158 "user.name=Test",
159 "-c",
160 "user.email=test@example.test",
161 "commit",
162 "--allow-empty",
163 "-m",
164 "initial",
165 ]);
166 yield* command(repo, ["worktree", "add", "-b", "t3/abcd1234", worktreePath]);
167 return { repo, worktreePath, command };
168 });
169 const modelSelection = {
170 instanceId: ProviderInstanceId.make("codex"),
171 model: "gpt-5.4",
172 } satisfies ModelSelection;
173 const CODEX_DRIVER = ProviderDriverKind.make("codex");
174 const isProviderEvent = Schema.is(ProviderAdapter.ProviderAdapterV2Event);
175
176 function threadCreatedEvent(
177 now: DateTime.Utc,
178 ): Effect.Effect<
179 OrchestrationV2DomainEvent,
180 IdAllocator.IdAllocatorV2Error,
181 IdAllocator.IdAllocatorV2
182 > {
183 return Effect.gen(function* () {
184 const idAllocator = yield* IdAllocator.IdAllocatorV2;
185 const projectId = yield* idAllocator.allocate.project({
186 fixtureName: "provider-event-ingestor",
187 });
188 const threadId = yield* idAllocator.allocate.thread({
189 fixtureName: "provider-event-ingestor",
190 projectId,
191 });
192 const providerThreadId = idAllocator.derive.providerThread({
193 driver: CODEX_DRIVER,
194 nativeThreadId: "native-thread",
195 });
196 const thread: OrchestrationV2AppThread = {
197 createdBy: "user",
198 creationSource: "web",
199 id: threadId,
200 projectId,
201 title: "Provider event ingestor",
202 providerInstanceId: modelSelection.instanceId,
203 modelSelection: modelSelection,
204 runtimeMode: "full-access",
205 interactionMode: "default",
206 branch: null,
207 worktreePath: null,
208 branchPullRequest: null,
209 activeOrderKey: null,
210 activeProviderThreadId: providerThreadId,
211 lineage: {
212 parentThreadId: null,
213 relationshipToParent: null,
214 rootThreadId: threadId,
215 },
216 forkedFrom: null,
217 createdAt: now,
218 updatedAt: now,
219 archivedAt: null,
220 settledOverride: null,
221 settledAt: null,
222 lastVisitedAt: null,
223 deletedAt: null,
224 };
225
226 return {
227 id: yield* idAllocator.allocate.event({ threadId }),
228 type: "thread.created",
229 threadId,
230 occurredAt: now,
231 payload: thread,
232 };
233 });
234 }
235
236 const layer = it.layer(layerTest);
237
238 it.effect("takes a hub's reset for an unreset usage_limit, then refreshes the snapshot", () =>
239 Effect.gen(function* () {
240 const refreshed = yield* Deferred.make<void>();
241 const adapter = yield* ProviderAdapter.ProviderAdapterV2.pipe(
242 Effect.provide(
243 Layer.mock(ProviderAdapter.ProviderAdapterV2)({
244 instanceId: ProviderInstanceId.make("cursor-work"),
245 driver: ProviderDriverKind.make("cursor"),
246 }),
247 ),
248 );
249 const instance: ProviderInstance = {
250 instanceId: ProviderInstanceId.make("cursor-work"),
251 driverKind: ProviderDriverKind.make("cursor"),
252 continuationIdentity: {
253 driverKind: ProviderDriverKind.make("cursor"),
254 continuationKey: "work",
255 },
256 enabled: true,
257 displayName: undefined,
258 orchestrationAdapter: adapter,
259 textGeneration: {
260 generateBranchName: () => Effect.die("unused"),
261 generateCommitMessage: () => Effect.die("unused"),
262 generatePrContent: () => Effect.die("unused"),
263 generateThreadTitle: () => Effect.die("unused"),
264 generateReviewGuide: () => Effect.die("unused"),
265 generateTranscriptionPostProcessing: () => Effect.die("unused"),
266 },
267 snapshot: {
268 refreshUsage: Deferred.succeed(refreshed, undefined).pipe(Effect.asVoid),
269 getSnapshot: Effect.succeed({
270 auth: { status: "authenticated", endpoint: "http://hub.test:8317" },
271 } as ServerProvider),
272 refresh: Effect.die("unused"),
273 resolveMaintenance: () => Effect.die("unused"),
274 applyUsageLimits: () => Effect.die("unused"),
275 streamChanges: Stream.empty,
276 },
277 };
278 const lookups: ProviderInstanceId[] = [];
279 const hubLookups: Array<[string, string, string | null]> = [];
280 const instances = Layer.mock(ProviderInstanceRegistry.ProviderInstanceRegistry)({
281 getInstance: (id) =>
282 Effect.sync(() => {
283 lookups.push(id);
284 return id === instance.instanceId ? instance : undefined;
285 }),
286 });
287 yield* Effect.gen(function* () {
288 const now = yield* DateTime.now;
289 const eventSink = yield* EventSink.EventSinkV2;
290 const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
291 const idAllocator = yield* IdAllocator.IdAllocatorV2;
292 const threadEvent = yield* threadCreatedEvent(now);
293 yield* eventSink.write({ events: [threadEvent] });
294 const input = {
295 providerSessionId: yield* idAllocator.allocate.providerSession({
296 providerInstanceId: instance.instanceId,
297 threadId: threadEvent.threadId,
298 }),
299 providerInstanceId: instance.instanceId,
300 threadId: threadEvent.threadId,
301 };
302 for (const failureClass of ["validation_error", "usage_limit"] as const) {
303 const stored = yield* ingestor.ingestNormalized({
304 ...input,
305 event: {
306 type: "turn.terminal",
307 driver: instance.driverKind,
308 providerThreadId: idAllocator.derive.providerThread({
309 driver: instance.driverKind,
310 nativeThreadId: "limited",
311 }),
312 providerTurnId: idAllocator.derive.providerTurn({
313 driver: instance.driverKind,
314 nativeTurnId: failureClass,
315 }),
316 runOrdinal: 1,
317 failureItemOrdinal: 1,
318 status: "failed",
319 failure: makeProviderFailure({
320 class: failureClass,
321 message: "Provider rejected turn.",
322 }),
323 threadDisposition: "reusable",
324 },
325 });
326 assert.equal(stored.length, 1);
327 if (failureClass === "validation_error") assert.deepEqual(lookups, []);
328 else {
329 const event = stored[0]?.event;
330 assert.equal(
331 event?.type === "turn-item.updated" && event.payload.type === "error"
332 ? event.payload.failure.resetAt
333 : undefined,
334 "2026-11-13T18:00:00.000Z",
335 );
336 }
337 }
338 assert.deepEqual(lookups, [instance.instanceId, instance.instanceId]);
339 assert.deepEqual(hubLookups, [["cursor", "http://hub.test:8317", "gpt-5.4"]]);
340 yield* Deferred.await(refreshed);
341 }).pipe(
342 Effect.provide(makeLayerTest(layerTestEventSink, instances)),
343 Effect.provideService(UsageLimitResetFallback, {
344 resetAt: (driver, endpoint, model) =>
345 Effect.sync(() => {
346 hubLookups.push([driver, endpoint, model]);
347 return "2026-11-13T18:00:00.000Z";
348 }),
349 }),
350 );
351 }),
352 );
353
354 it.effect("records accepted billed turn usage once without billing the context window", () => {
355 const recorded: Array<ProviderUsageAnalytics.ProviderTurnReport> = [];
356 const layerAnalytics = Layer.succeed(ProviderUsageAnalytics.ProviderTurnAnalytics, {
357 runStarted: () => Effect.void,
358 turnCompleted: (report) =>
359 Effect.sync(() => {
360 recorded.push(report);
361 }),
362 runSettled: () => Effect.void,
363 });
364 return Effect.gen(function* () {
365 const now = yield* DateTime.now;
366 const eventSink = yield* EventSink.EventSinkV2;
367 const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
368 const idAllocator = yield* IdAllocator.IdAllocatorV2;
369 const threadEvent = yield* threadCreatedEvent(now);
370 yield* eventSink.write({ events: [threadEvent] });
371 const providerSessionId = yield* idAllocator.allocate.providerSession({
372 providerInstanceId: modelSelection.instanceId,
373 threadId: threadEvent.threadId,
374 });
375 const providerThreadId = idAllocator.derive.providerThread({
376 driver: CODEX_DRIVER,
377 nativeThreadId: "billing-thread",
378 });
379 const providerTurn = {
380 id: idAllocator.derive.providerTurn({ driver: CODEX_DRIVER, nativeTurnId: "billing-turn" }),
381 providerThreadId,
382 nodeId: NodeId.make("node:billing-turn"),
383 runAttemptId: null,
384 nativeTurnRef: null,
385 ordinal: 1,
386 status: "completed" as const,
387 startedAt: now,
388 completedAt: DateTime.makeUnsafe(DateTime.toEpochMillis(now) + 120),
389 tokenUsage: {
390 inputTokens: 4000,
391 cachedInputTokens: 3000,
392 outputTokens: 100,
393 usedTokens: 4100,
394 updatedAt: DateTime.formatIso(now),
395 },
396 turnTokenUsage: {
397 usageStatus: "complete" as const,
398 usageScope: "main_agent" as const,
399 hasSubagents: false,
400 inputTokens: 40,
401 cachedInputTokens: 30,
402 outputTokens: 10,
403 },
404 };
405 const input = {
406 providerSessionId,
407 providerInstanceId: modelSelection.instanceId,
408 threadId: threadEvent.threadId,
409 analyticsContext: {
410 modelSelection,
411 runtimeMode: "full-access" as const,
412 interactionMode: "default" as const,
413 origin: {
414 createdBy: "agent" as const,
415 creationSource: "provider" as const,
416 scheduled: false,
417 subagent: true,
418 },
419 },
420 event: { type: "provider_turn.updated" as const, driver: CODEX_DRIVER, providerTurn },
421 };
422 yield* ingestor.ingestNormalized(input);
423 yield* ingestor.ingestNormalized(input);
424 const ignored = yield* ingestor.ingestNormalized({
425 ...input,
426 event: {
427 ...input.event,
428 providerTurn: {
429 ...providerTurn,
430 id: idAllocator.derive.providerTurn({
431 driver: CODEX_DRIVER,
432 nativeTurnId: "stale-billing-turn",
433 }),
434 },
435 },
436 writeIfRunCurrent: {
437 runId: RunId.make("missing-run"),
438 activeAttemptId: RunAttemptId.make("stale-attempt"),
439 expectedStatus: "running",
440 },
441 });
442 assert.isEmpty(ignored);
443 assert.lengthOf(recorded, 1);
444 assert.deepEqual(recorded[0], {
445 driver: CODEX_DRIVER,
446 providerTurn,
447 context: input.analyticsContext,
448 });
449 }).pipe(Effect.provide(layerTest.pipe(Layer.provide(layerAnalytics))));
450 });
451
452 it.effect(
453 "reports invalid provider questions as typed errors and still ingests valid questions",
454 () =>
455 Effect.gen(function* () {
456 const now = yield* DateTime.now;
457 const eventSink = yield* EventSink.EventSinkV2;
458 const eventStore = yield* EventStore.EventStoreV2;
459 const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
460 const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
461 const idAllocator = yield* IdAllocator.IdAllocatorV2;
462 const threadEvent = yield* threadCreatedEvent(now);
463 const threadId = threadEvent.threadId;
464 yield* eventSink.write({ events: [threadEvent] });
465 const providerInstanceId = ProviderInstanceId.make("pi");
466 const providerSessionId = yield* idAllocator.allocate.providerSession({
467 providerInstanceId,
468 threadId,
469 });
470 const question = { id: "input", header: "Enter a value", question: "", options: [] };
471 const item = {
472 id: TurnItemId.make(`${threadId}:question`),
473 threadId,
474 runId: null,
475 nodeId: null,
476 providerThreadId: null,
477 providerTurnId: null,
478 nativeItemRef: null,
479 parentItemId: null,
480 ordinal: 1,
481 status: "waiting",
482 title: "Enter a value",
483 startedAt: now,
484 completedAt: null,
485 updatedAt: now,
486 type: "user_input_request",
487 requestId: RuntimeRequestId.make(`${threadId}:question`),
488 questions: [question],
489 } satisfies OrchestrationV2TurnItem;
490 const event = {
491 type: "turn_item.updated",
492 driver: ProviderDriverKind.make("pi"),
493 turnItem: item,
494 } satisfies ProviderAdapter.ProviderAdapterV2Event;
495 const input = { providerSessionId, providerInstanceId, threadId, event };
496
497 assert.isFalse(isProviderEvent(event));
498 const error = yield* ingestor.ingestNormalized(input).pipe(Effect.flip);
499 assert.instanceOf(error, ProviderEventIngestor.ProviderEventNormalizeError);
500 assert.equal(
501 error.message,
502 `Failed to normalize provider event turn_item.updated for thread ${threadId}.`,
503 );
504 assert.instanceOf(error.cause, Schema.SchemaError);
505 assert.equal(yield* eventStore.latestSequence({ threadId }), 1);
506 assert.isEmpty((yield* projectionStore.getThreadProjection(threadId)).turnItems);
507
508 const validEvent = {
509 ...event,
510 turnItem: { ...item, questions: [{ ...question, question: "Enter a value" }] },
511 };
512 assert.isTrue(isProviderEvent(validEvent));
513 const stored = yield* ingestor.ingestNormalized({ ...input, event: validEvent });
514 assert.lengthOf(stored, 1);
515 assert.equal(stored[0]?.event.type, "turn-item.updated");
516 assert.deepEqual((yield* projectionStore.getThreadProjection(threadId)).turnItems, [
517 validEvent.turnItem,
518 ]);
519 }).pipe(Effect.provide(layerTest)),
520 );
521
522 layer("ProviderEventIngestorV2", (it) => {
523 it.effect("normalizes provider events through the real event log and projection store", () =>
524 Effect.gen(function* () {
525 const now = yield* DateTime.now;
526 const eventSink = yield* EventSink.EventSinkV2;
527 const eventStore = yield* EventStore.EventStoreV2;
528 const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
529 const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
530 const idAllocator = yield* IdAllocator.IdAllocatorV2;
531 const threadEvent = yield* threadCreatedEvent(now);
532 const providerSessionId = yield* idAllocator.allocate.providerSession({
533 providerInstanceId: modelSelection.instanceId,
534 threadId: threadEvent.threadId,
535 });
536 const providerThread: OrchestrationV2ProviderThread = {
537 id: idAllocator.derive.providerThread({
538 driver: CODEX_DRIVER,
539 nativeThreadId: "native-thread",
540 }),
541 driver: CODEX_DRIVER,
542 providerInstanceId: modelSelection.instanceId,
543 providerSessionId,
544 appThreadId: threadEvent.threadId,
545 ownerNodeId: null,
546 nativeThreadRef: {
547 driver: CODEX_DRIVER,
548 nativeId: "native-thread",
549 strength: "strong",
550 },
551 nativeConversationHeadRef: null,
552 status: "idle",
553 firstRunOrdinal: null,
554 lastRunOrdinal: null,
555 handoffIds: [],
556 forkedFrom: null,
557 createdAt: now,
558 updatedAt: now,
559 };
560
561 yield* eventSink.write({ events: [threadEvent] });
562 const storedEvents = yield* ingestor.ingestNormalized({
563 providerSessionId,
564 providerInstanceId: modelSelection.instanceId,
565 threadId: threadEvent.threadId,
566 event: {
567 type: "provider_thread.updated",
568 driver: CODEX_DRIVER,
569 providerThread,
570 },
571 });
572
573 const projection = yield* projectionStore.getThreadProjection(threadEvent.threadId);
574 const storedDomainEvents = yield* eventStore.read({}).pipe(Stream.runCollect);
575 const afterFirstEvent = yield* eventStore
576 .read({ afterSequence: 1, threadId: threadEvent.threadId })
577 .pipe(Stream.runCollect);
578 const latestThreadSequence = yield* eventStore.latestSequence({
579 threadId: threadEvent.threadId,
580 });
581
582 assert.equal(storedEvents.length, 1);
583 assert.equal(storedEvents[0]?.event.type, "provider-thread.updated");
584 assert.deepEqual(
585 projection.providerThreads.map((thread) => thread.id),
586 [providerThread.id],
587 );
588 assert.deepEqual(
589 Array.from(storedDomainEvents).map((stored) => stored.event.type),
590 ["thread.created", "provider-thread.updated"],
591 );
592 assert.deepEqual(
593 Array.from(storedDomainEvents).map((stored) => stored.sequence),
594 [1, 2],
595 );
596 assert.deepEqual(
597 Array.from(afterFirstEvent).map((stored) => stored.event.type),
598 ["provider-thread.updated"],
599 );
600 assert.equal(latestThreadSequence, 2);
601 }),
602 );
603
604 it.effect("skips a node update only when it matches the stored node", () =>
605 Effect.gen(function* () {
606 const now = yield* DateTime.now;
607 const eventSink = yield* EventSink.EventSinkV2;
608 const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
609 const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
610 const idAllocator = yield* IdAllocator.IdAllocatorV2;
611 const threadEvent = yield* threadCreatedEvent(now);
612 const threadId = threadEvent.threadId;
613 yield* eventSink.write({ events: [threadEvent] });
614 const providerSessionId = yield* idAllocator.allocate.providerSession({
615 providerInstanceId: modelSelection.instanceId,
616 threadId,
617 });
618 const nodeId = NodeId.make(`${threadId}:approval`);
619 const node: OrchestrationV2ExecutionNode = {
620 id: nodeId,
621 threadId,
622 runId: null,
623 parentNodeId: null,
624 rootNodeId: nodeId,
625 kind: "approval_request",
626 status: "waiting",
627 countsForRun: false,
628 providerThreadId: null,
629 providerTurnId: null,
630 nativeItemRef: null,
631 runtimeRequestId: null,
632 checkpointScopeId: null,
633 startedAt: now,
634 completedAt: null,
635 };
636 const ingestNode = (sent: OrchestrationV2ExecutionNode) =>
637 ingestor.ingestNormalized({
638 providerSessionId,
639 providerInstanceId: modelSelection.instanceId,
640 threadId,
641 event: { type: "node.updated", driver: CODEX_DRIVER, node: sent },
642 });
643
644 yield* eventSink.withProviderBatch(
645 Effect.gen(function* () {
646 assert.lengthOf(yield* ingestNode(node), 1);
647 assert.isEmpty(yield* ingestNode({ ...node }));
648 }),
649 );
650 yield* eventSink.write({
651 events: [
652 {
653 id: yield* idAllocator.allocate.event({ threadId }),
654 type: "node.updated",
655 threadId,
656 occurredAt: now,
657 payload: { ...node, status: "completed", completedAt: now },
658 },
659 ],
660 });
661 assert.lengthOf(yield* ingestNode(node), 1);
662 const projection = yield* projectionStore.getThreadProjection(threadId);
663 assert.equal(projection.nodes.find((entry) => entry.id === nodeId)?.status, "waiting");
664 }),
665 );
666
667 it.effect("rolls back runs a provider rewound off its active branch", () =>
668 Effect.gen(function* () {
669 const now = yield* DateTime.now;
670 const eventSink = yield* EventSink.EventSinkV2;
671 const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
672 const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
673 const idAllocator = yield* IdAllocator.IdAllocatorV2;
674 const threadEvent = yield* threadCreatedEvent(now);
675 const threadId = threadEvent.threadId;
676 const providerSessionId = yield* idAllocator.allocate.providerSession({
677 providerInstanceId: modelSelection.instanceId,
678 threadId,
679 });
680 const rewoundThreadId = idAllocator.derive.providerThread({
681 driver: CODEX_DRIVER,
682 nativeThreadId: "rewound-thread",
683 });
684 const otherThreadId = idAllocator.derive.providerThread({
685 driver: CODEX_DRIVER,
686 nativeThreadId: "other-thread",
687 });
688 const scopeId = CheckpointScopeId.make("checkpoint-scope:rewound");
689 const seedRun = Effect.fnUntraced(function* (input: {
690 readonly ordinal: number;
691 readonly providerThreadId: ProviderThreadId;
692 readonly status: "completed" | "interrupted" | "running";
693 readonly nativeTurnId: string | null;
694 }) {
695 const runId = RunId.make(`run:rewound:${input.ordinal}`);
696 const attemptId = RunAttemptId.make(`attempt:rewound:${input.ordinal}`);
697 const nodeId = NodeId.make(`node:rewound:${input.ordinal}`);
698 const providerTurnId = ProviderTurnId.make(`provider-turn:rewound:${input.ordinal}`);
699 const completedAt = input.status === "running" ? null : now;
700 const eventId = () => idAllocator.allocate.event({ threadId });
701 const events: Array<OrchestrationV2DomainEvent> = [
702 {
703 id: yield* eventId(),
704 type: "run.created",
705 threadId,
706 occurredAt: now,
707 payload: {
708 id: runId,
709 threadId,
710 ordinal: input.ordinal,
711 providerInstanceId: modelSelection.instanceId,
712 modelSelection,
713 providerThreadId: input.providerThreadId,
714 userMessageId: MessageId.make(`message:rewound:${input.ordinal}`),
715 rootNodeId: nodeId,
716 activeAttemptId: attemptId,
717 status: input.status,
718 requestedAt: now,
719 startedAt: now,
720 completedAt,
721 checkpointId: null,
722 contextHandoffId: null,
723 },
724 },
725 {
726 id: yield* eventId(),
727 type: "run-attempt.created",
728 threadId,
729 occurredAt: now,
730 payload: {
731 id: attemptId,
732 runId,
733 attemptOrdinal: 1,
734 rootNodeId: nodeId,
735 providerInstanceId: modelSelection.instanceId,
736 providerThreadId: input.providerThreadId,
737 providerTurnId,
738 reason: "initial",
739 status: input.status,
740 startedAt: now,
741 completedAt,
742 },
743 },
744 {
745 id: yield* eventId(),
746 type: "provider-turn.updated",
747 threadId,
748 occurredAt: now,
749 payload: {
750 id: providerTurnId,
751 providerThreadId: input.providerThreadId,
752 nodeId,
753 runAttemptId: attemptId,
754 nativeTurnRef:
755 input.nativeTurnId === null
756 ? {
757 driver: CODEX_DRIVER,
758 nativeId: `synthetic:${input.ordinal}`,
759 strength: "weak",
760 }
761 : { driver: CODEX_DRIVER, nativeId: input.nativeTurnId, strength: "strong" },
762 ordinal: input.ordinal,
763 status: input.status,
764 startedAt: now,
765 completedAt,
766 },
767 },
768 {
769 id: yield* eventId(),
770 type: "node.updated",
771 threadId,
772 occurredAt: now,
773 payload: {
774 id: nodeId,
775 threadId,
776 runId,
777 parentNodeId: null,
778 rootNodeId: nodeId,
779 kind: "root_turn",
780 status: input.status,
781 countsForRun: true,
782 providerThreadId: input.providerThreadId,
783 providerTurnId,
784 nativeItemRef: null,
785 runtimeRequestId: null,
786 checkpointScopeId: scopeId,
787 startedAt: now,
788 completedAt,
789 },
790 },
791 ...(input.status === "completed"
792 ? [
793 {
794 id: yield* eventId(),
795 type: "checkpoint.captured" as const,
796 threadId,
797 occurredAt: now,
798 payload: {
799 id: CheckpointId.make(`checkpoint:rewound:${input.ordinal}`),
800 threadId,
801 scopeId,
802 runId,
803 nodeId,
804 parentCheckpointId: null,
805 ordinalWithinScope: input.ordinal,
806 appRunOrdinal: input.ordinal,
807 ref: CheckpointRef.make(`refs/t3/checkpoints/rewound/${input.ordinal}`),
808 status: "ready" as const,
809 files: [],
810 capturedAt: now,
811 },
812 },
813 ]
814 : []),
815 ];
816 yield* eventSink.write({ events });
817 return runId;
818 });
819
820 yield* eventSink.write({ events: [threadEvent] });
821 const otherProviderRun = yield* seedRun({
822 ordinal: 1,
823 providerThreadId: otherThreadId,
824 status: "completed",
825 nativeTurnId: "other-turn",
826 });
827 const keptRun = yield* seedRun({
828 ordinal: 2,
829 providerThreadId: rewoundThreadId,
830 status: "completed",
831 nativeTurnId: "kept-turn",
832 });
833 const abandonedRun = yield* seedRun({
834 ordinal: 3,
835 providerThreadId: rewoundThreadId,
836 status: "completed",
837 nativeTurnId: "abandoned-turn",
838 });
839 const unlocatedRun = yield* seedRun({
840 ordinal: 4,
841 providerThreadId: rewoundThreadId,
842 status: "interrupted",
843 nativeTurnId: null,
844 });
845 const rewindingRun = yield* seedRun({
846 ordinal: 5,
847 providerThreadId: rewoundThreadId,
848 status: "running",
849 nativeTurnId: null,
850 });
851
852 yield* ingestor.ingestNormalized({
853 providerSessionId,
854 providerInstanceId: modelSelection.instanceId,
855 threadId,
856 runId: rewindingRun,
857 event: {
858 type: "provider_thread.updated",
859 driver: CODEX_DRIVER,
860 providerThread: {
861 id: rewoundThreadId,
862 driver: CODEX_DRIVER,
863 providerInstanceId: modelSelection.instanceId,
864 providerSessionId,
865 appThreadId: threadId,
866 ownerNodeId: null,
867 nativeThreadRef: {
868 driver: CODEX_DRIVER,
869 nativeId: "rewound-thread",
870 strength: "strong",
871 },
872 nativeConversationHeadRef: {
873 driver: CODEX_DRIVER,
874 nativeId: "kept-turn-reply",
875 strength: "strong",
876 },
877 status: "idle",
878 firstRunOrdinal: 2,
879 lastRunOrdinal: 5,
880 handoffIds: [],
881 forkedFrom: null,
882 createdAt: now,
883 updatedAt: now,
884 },
885 retainedNativeTurnIds: ["turn-started-outside-t3", "kept-turn"],
886 },
887 });
888
889 const projection = yield* projectionStore.getThreadProjection(threadId);
890 const runIds = [otherProviderRun, keptRun, abandonedRun, unlocatedRun, rewindingRun];
891 assert.deepEqual(
892 runIds.map((runId) => projection.runs.find((run) => run.id === runId)?.status),
893 ["completed", "completed", "rolled_back", "rolled_back", "running"],
894 );
895 assert.deepEqual(
896 runIds.map((runId) => projection.nodes.find((node) => node.runId === runId)?.status),
897 ["completed", "completed", "rolled_back", "rolled_back", "running"],
898 );
899 assert.deepEqual(
900 [otherProviderRun, keptRun, abandonedRun].map(
901 (runId) =>
902 projection.checkpoints.find((checkpoint) => checkpoint.runId === runId)?.status,
903 ),
904 ["ready", "ready", "stale"],
905 );
906 assert.equal(
907 projection.providerThreads.find((thread) => thread.id === rewoundThreadId)?.lastRunOrdinal,
908 5,
909 );
910 }),
911 );
912
913 it.effect(
914 "treats successful provider terminal markers as non-persisted orchestration control signals",
915 () =>
916 Effect.gen(function* () {
917 const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
918 const idAllocator = yield* IdAllocator.IdAllocatorV2;
919 const projectId = yield* idAllocator.allocate.project({
920 fixtureName: "provider-event-terminal",
921 });
922 const threadId = yield* idAllocator.allocate.thread({
923 fixtureName: "provider-event-terminal",
924 projectId,
925 });
926 const providerSessionId = yield* idAllocator.allocate.providerSession({
927 providerInstanceId: modelSelection.instanceId,
928 threadId,
929 });
930 const normalized = yield* ingestor.normalize({
931 providerSessionId,
932 providerInstanceId: modelSelection.instanceId,
933 threadId,
934 event: {
935 type: "turn.terminal",
936 driver: CODEX_DRIVER,
937 providerThreadId: idAllocator.derive.providerThread({
938 driver: CODEX_DRIVER,
939 nativeThreadId: "native-thread",
940 }),
941 providerTurnId: idAllocator.derive.providerTurn({
942 driver: CODEX_DRIVER,
943 nativeTurnId: "native-turn",
944 }),
945 runOrdinal: 1,
946 status: "completed",
947 failure: null,
948 threadDisposition: "reusable",
949 },
950 });
951
952 assert.deepEqual(normalized, []);
953 }),
954 );
955
956 it.effect("persists an interrupted run's inherited terminal through the live run router", () =>
957 Effect.gen(function* () {
958 const now = yield* DateTime.now;
959 const eventSink = yield* EventSink.EventSinkV2;
960 const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
961 const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
962 const idAllocator = yield* IdAllocator.IdAllocatorV2;
963 const threadEvent = yield* threadCreatedEvent(now);
964 const priorRunId = RunId.make("run:provider-event-inherited:prior");
965 const currentRunId = RunId.make("run:provider-event-inherited:current");
966 const itemId = TurnItemId.make("turn-item:provider-event-inherited");
967 const providerSessionId = yield* idAllocator.allocate.providerSession({
968 providerInstanceId: modelSelection.instanceId,
969 threadId: threadEvent.threadId,
970 });
971 const providerThreadId = idAllocator.derive.providerThread({
972 driver: CODEX_DRIVER,
973 nativeThreadId: "native-thread-inherited",
974 });
975 const providerTurnId = idAllocator.derive.providerTurn({
976 driver: CODEX_DRIVER,
977 nativeTurnId: "native-turn-inherited",
978 });
979 const runningItem = {
980 id: itemId,
981 threadId: threadEvent.threadId,
982 runId: priorRunId,
983 nodeId: NodeId.make("node:provider-event-inherited"),
984 providerThreadId,
985 providerTurnId,
986 nativeItemRef: null,
987 parentItemId: null,
988 ordinal: 101,
989 status: "running",
990 title: "Inherited background command",
991 startedAt: now,
992 completedAt: null,
993 updatedAt: now,
994 type: "command_execution",
995 input: "sleep 60",
996 } satisfies OrchestrationV2TurnItem;
997 const terminalItem = {
998 ...runningItem,
999 status: "completed" as const,
1000 completedAt: now,
1001 updatedAt: now,
1002 };
1003
1004 yield* eventSink.write({ events: [threadEvent] });
1005 yield* ingestor.ingestNormalized({
1006 providerSessionId,
1007 providerInstanceId: modelSelection.instanceId,
1008 threadId: threadEvent.threadId,
1009 runId: priorRunId,
1010 event: { type: "turn_item.updated", driver: CODEX_DRIVER, turnItem: runningItem },
1011 });
1012
1013 const identity: ProviderEventRouteIdentity = {
1014 threadId: threadEvent.threadId,
1015 runId: currentRunId,
1016 attemptId: RunAttemptId.make("attempt:provider-event-inherited:current"),
1017 providerThreadId,
1018 };
1019 const inheritedBackgroundTurnItems = selectInheritedBackgroundTurnItems({
1020 threadId: threadEvent.threadId,
1021 currentProviderThreadId: providerThreadId,
1022 currentRunOrdinal: 2,
1023 runs: [
1024 {
1025 id: priorRunId,
1026 threadId: threadEvent.threadId,
1027 ordinal: 1,
1028 status: "interrupted",
1029 } as OrchestrationV2Run,
1030 {
1031 id: currentRunId,
1032 threadId: threadEvent.threadId,
1033 ordinal: 2,
1034 status: "running",
1035 } as OrchestrationV2Run,
1036 ],
1037 turnItems: [runningItem],
1038 });
1039 const routeState = makeProviderEventRoutingState({
1040 identity,
1041 inheritedBackgroundTurnItems,
1042 providerTurnId: null,
1043 });
1044 const terminalEvent = {
1045 type: "turn_item.updated",
1046 driver: CODEX_DRIVER,
1047 turnItem: terminalItem,
1048 } as const;
1049 const [accepted] = routeProviderEvent(terminalEvent, identity, routeState);
1050 assert.isTrue(accepted);
1051
1052 const stored = yield* ingestor.ingestNormalized({
1053 providerSessionId,
1054 providerInstanceId: modelSelection.instanceId,
1055 threadId: threadEvent.threadId,
1056 runId: currentRunId,
1057 event: terminalEvent,
1058 });
1059 const projection = yield* projectionStore.getThreadProjection(threadEvent.threadId);
1060 const persisted = projection.turnItems.find((item) => item.id === itemId);
1061
1062 assert.equal(stored.length, 1);
1063 assert.equal(stored[0]?.event.type, "turn-item.updated");
1064 assert.equal(persisted?.runId, priorRunId);
1065 assert.equal(persisted?.threadId, threadEvent.threadId);
1066 assert.equal(persisted?.status, "completed");
1067 }),
1068 );
1069
1070 it.effect("persists a completed run's late background terminal exactly once", () =>
1071 Effect.gen(function* () {
1072 const now = yield* DateTime.now;
1073 const eventSink = yield* EventSink.EventSinkV2;
1074 const eventStore = yield* EventStore.EventStoreV2;
1075 const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
1076 const idAllocator = yield* IdAllocator.IdAllocatorV2;
1077 const threadEvent = yield* threadCreatedEvent(now);
1078 const priorRunId = RunId.make("run:provider-event-completed:prior");
1079 const currentRunId = RunId.make("run:provider-event-completed:current");
1080 const itemId = TurnItemId.make("turn-item:provider-event-completed");
1081 const providerSessionId = yield* idAllocator.allocate.providerSession({
1082 providerInstanceId: modelSelection.instanceId,
1083 threadId: threadEvent.threadId,
1084 });
1085 const providerThreadId = idAllocator.derive.providerThread({
1086 driver: CODEX_DRIVER,
1087 nativeThreadId: "native-thread-completed",
1088 });
1089 const providerTurnId = idAllocator.derive.providerTurn({
1090 driver: CODEX_DRIVER,
1091 nativeTurnId: "native-turn-completed",
1092 });
1093 const runningItem = {
1094 id: itemId,
1095 threadId: threadEvent.threadId,
1096 runId: priorRunId,
1097 nodeId: NodeId.make("node:provider-event-completed"),
1098 providerThreadId,
1099 providerTurnId,
1100 nativeItemRef: null,
1101 parentItemId: null,
1102 ordinal: 101,
1103 status: "running",
1104 title: "Completed run background command",
1105 startedAt: now,
1106 completedAt: null,
1107 updatedAt: now,
1108 type: "command_execution",
1109 input: "sleep 60",
1110 } satisfies OrchestrationV2TurnItem;
1111 const terminalEvent = {
1112 type: "turn_item.updated",
1113 driver: CODEX_DRIVER,
1114 turnItem: {
1115 ...runningItem,
1116 status: "completed" as const,
1117 completedAt: now,
1118 updatedAt: now,
1119 },
1120 } as const;
1121
1122 yield* eventSink.write({ events: [threadEvent] });
1123 yield* ingestor.ingestNormalized({
1124 providerSessionId,
1125 providerInstanceId: modelSelection.instanceId,
1126 threadId: threadEvent.threadId,
1127 runId: priorRunId,
1128 event: { type: "turn_item.updated", driver: CODEX_DRIVER, turnItem: runningItem },
1129 });
1130
1131 const priorIdentity: ProviderEventRouteIdentity = {
1132 threadId: threadEvent.threadId,
1133 runId: priorRunId,
1134 attemptId: RunAttemptId.make("attempt:provider-event-completed:prior"),
1135 providerThreadId,
1136 };
1137 const currentIdentity: ProviderEventRouteIdentity = {
1138 threadId: threadEvent.threadId,
1139 runId: currentRunId,
1140 attemptId: RunAttemptId.make("attempt:provider-event-completed:current"),
1141 providerThreadId,
1142 };
1143 const inheritedBackgroundTurnItems = selectInheritedBackgroundTurnItems({
1144 threadId: threadEvent.threadId,
1145 currentProviderThreadId: providerThreadId,
1146 currentRunOrdinal: 2,
1147 runs: [
1148 {
1149 id: priorRunId,
1150 threadId: threadEvent.threadId,
1151 ordinal: 1,
1152 status: "completed",
1153 } as OrchestrationV2Run,
1154 {
1155 id: currentRunId,
1156 threadId: threadEvent.threadId,
1157 ordinal: 2,
1158 status: "running",
1159 } as OrchestrationV2Run,
1160 ],
1161 turnItems: [runningItem],
1162 });
1163 const routers = [
1164 {
1165 identity: priorIdentity,
1166 state: makeProviderEventRoutingState({
1167 identity: priorIdentity,
1168 providerTurnId: providerTurnId,
1169 }),
1170 },
1171 {
1172 identity: currentIdentity,
1173 state: makeProviderEventRoutingState({
1174 identity: currentIdentity,
1175 inheritedBackgroundTurnItems,
1176 providerTurnId: null,
1177 }),
1178 },
1179 ];
1180 const acceptedRouters = routers.filter(
1181 ({ identity, state }) => routeProviderEvent(terminalEvent, identity, state)[0],
1182 );
1183
1184 yield* Effect.forEach(
1185 acceptedRouters,
1186 ({ identity }) =>
1187 ingestor.ingestNormalized({
1188 providerSessionId,
1189 providerInstanceId: modelSelection.instanceId,
1190 threadId: threadEvent.threadId,
1191 runId: identity.runId,
1192 event: terminalEvent,
1193 }),
1194 { concurrency: 1 },
1195 );
1196
1197 const storedEvents = yield* eventStore
1198 .read({ threadId: threadEvent.threadId })
1199 .pipe(Stream.runCollect);
1200 const storedTerminals = Array.from(storedEvents).filter(
1201 (stored) =>
1202 stored.event.type === "turn-item.updated" &&
1203 stored.event.payload.id === itemId &&
1204 stored.event.payload.status === "completed",
1205 );
1206
1207 assert.equal(storedTerminals.length, 1);
1208 assert.equal(acceptedRouters.length, 1);
1209 assert.equal(acceptedRouters[0]?.identity.runId, priorRunId);
1210 }),
1211 );
1212
1213 it.effect.each(["interrupted", "control"] as const)(
1214 "dismisses only native questions when a provider turn ends with %s",
1215 (terminal) =>
1216 Effect.gen(function* () {
1217 const now = yield* DateTime.now;
1218 const eventSink = yield* EventSink.EventSinkV2;
1219 const eventStore = yield* EventStore.EventStoreV2;
1220 const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
1221 const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
1222 const idAllocator = yield* IdAllocator.IdAllocatorV2;
1223 const threadEvent = yield* threadCreatedEvent(now);
1224 const threadId = threadEvent.threadId;
1225 const providerSessionId = yield* idAllocator.allocate.providerSession({
1226 providerInstanceId: modelSelection.instanceId,
1227 threadId,
1228 });
1229 const providerThreadId = idAllocator.derive.providerThread({
1230 driver: CODEX_DRIVER,
1231 nativeThreadId: `${threadId}:questions`,
1232 });
1233 const providerTurnId = idAllocator.derive.providerTurn({
1234 driver: CODEX_DRIVER,
1235 nativeTurnId: `${threadId}:questions`,
1236 });
1237 const otherTurnId = idAllocator.derive.providerTurn({
1238 driver: CODEX_DRIVER,
1239 nativeTurnId: `${threadId}:other-turn`,
1240 });
1241 const specs = [
1242 { key: "native", responseCapability: { type: "live", providerSessionId } },
1243 {
1244 key: "unavailable",
1245 responseCapability: { type: "not_resumable", reason: "Turn ended" },
1246 },
1247 { key: "message", responseCapability: { type: "message" } },
1248 { key: "answered", responseCapability: { type: "live", providerSessionId } },
1249 { key: "other-turn", responseCapability: { type: "live", providerSessionId } },
1250 { key: "approval", responseCapability: { type: "live", providerSessionId } },
1251 { key: "retained", responseCapability: { type: "live", providerSessionId } },
1252 ] as const;
1253 const fixtures = specs.map((spec, ordinal) => {
1254 const nodeId = NodeId.make(`${threadId}:${spec.key}`);
1255 const resolved = spec.key === "answered";
1256 const request: OrchestrationV2RuntimeRequest = {
1257 id: RuntimeRequestId.make(`${threadId}:${spec.key}`),
1258 nodeId,
1259 providerTurnId: spec.key === "other-turn" ? otherTurnId : providerTurnId,
1260 nativeRequestRef: null,
1261 kind: spec.key === "approval" ? "command" : "user_input",
1262 status: resolved ? "resolved" : "pending",
1263 responseCapability: spec.responseCapability,
1264 createdAt: now,
1265 resolvedAt: resolved ? now : null,
1266 };
1267 const node: OrchestrationV2ExecutionNode = {
1268 id: nodeId,
1269 threadId,
1270 runId: null,
1271 parentNodeId: null,
1272 rootNodeId: nodeId,
1273 kind: spec.key === "approval" ? "approval_request" : "user_input_request",
1274 status: resolved ? "completed" : "waiting",
1275 countsForRun: false,
1276 providerThreadId,
1277 providerTurnId: request.providerTurnId,
1278 nativeItemRef: null,
1279 runtimeRequestId: request.id,
1280 checkpointScopeId: null,
1281 startedAt: now,
1282 completedAt: resolved ? now : null,
1283 };
1284 const item: OrchestrationV2TurnItem = {
1285 id: TurnItemId.make(`${threadId}:${spec.key}`),
1286 threadId,
1287 runId: null,
1288 nodeId,
1289 providerThreadId,
1290 providerTurnId: request.providerTurnId,
1291 nativeItemRef: null,
1292 parentItemId: null,
1293 ordinal,
1294 status: resolved ? "completed" : "waiting",
1295 title: "Which option?",
1296 startedAt: now,
1297 completedAt: resolved ? now : null,
1298 updatedAt: now,
1299 ...(spec.key === "approval"
1300 ? { type: "approval_request", requestId: request.id, requestKind: "command" }
1301 : {
1302 type: "user_input_request",
1303 requestId: request.id,
1304 questions: [],
1305 ...(spec.key === "message" ? { responseMode: "message" as const } : {}),
1306 }),
1307 };
1308 return { key: spec.key, request, node, item };
1309 });
1310 const seedEvents: Array<OrchestrationV2DomainEvent> = [threadEvent];
1311 for (const fixture of fixtures) {
1312 for (const payload of [
1313 { type: "runtime-request.updated" as const, payload: fixture.request },
1314 { type: "node.updated" as const, payload: fixture.node },
1315 { type: "turn-item.updated" as const, payload: fixture.item },
1316 ]) {
1317 seedEvents.push({
1318 id: yield* idAllocator.allocate.event({ threadId }),
1319 threadId,
1320 occurredAt: now,
1321 ...payload,
1322 });
1323 }
1324 }
1325 yield* eventSink.write({ events: seedEvents });
1326 const retainedRuntimeRequestIds = [RuntimeRequestId.make(`${threadId}:retained`)];
1327 const input = {
1328 providerSessionId,
1329 providerInstanceId: modelSelection.instanceId,
1330 threadId,
1331 event:
1332 terminal === "control"
1333 ? {
1334 type: "turn.terminal" as const,
1335 driver: CODEX_DRIVER,
1336 providerThreadId,
1337 providerTurnId,
1338 runOrdinal: 1,
1339 status: "completed" as const,
1340 failure: null,
1341 threadDisposition: "reusable" as const,
1342 retainedRuntimeRequestIds,
1343 }
1344 : {
1345 type: "provider_turn.updated" as const,
1346 driver: CODEX_DRIVER,
1347 retainedRuntimeRequestIds,
1348 providerTurn: {
1349 id: providerTurnId,
1350 providerThreadId,
1351 nodeId: NodeId.make(`${threadId}:root`),
1352 runAttemptId: null,
1353 nativeTurnRef: null,
1354 ordinal: 1,
1355 status: terminal,
1356 startedAt: now,
1357 completedAt: now,
1358 },
1359 },
1360 };
1361 const stored = yield* ingestor.ingestNormalized(input);
1362 const projection = yield* projectionStore.getThreadProjection(threadId);
1363 for (const fixture of fixtures) {
1364 const closed = fixture.key === "native" || fixture.key === "unavailable";
1365 const request = projection.runtimeRequests.find(
1366 (item) => item.id === fixture.request.id,
1367 )!;
1368 const node = projection.nodes.find((item) => item.id === fixture.node.id)!;
1369 const item = projection.turnItems.find((item) => item.id === fixture.item.id)!;
1370 if (closed) {
1371 assert.equal(request.status, "cancelled");
1372 assert.isNotNull(request.resolvedAt);
1373 assert.equal(node.status, "cancelled");
1374 assert.isNotNull(node.completedAt);
1375 assert.equal(item.status, "cancelled");
1376 assert.isNotNull(item.completedAt);
1377 } else {
1378 assert.equal(request.status, fixture.request.status);
1379 assert.equal(node.status, fixture.node.status);
1380 assert.equal(item.status, fixture.item.status);
1381 }
1382 }
1383 assert.equal(
1384 stored.filter((entry) => entry.event.type === "runtime-request.updated").length,
1385 2,
1386 );
1387 assert.deepEqual(
1388 (yield* projectionStore.getPendingNativeUserInputs(
1389 threadId,
1390 providerTurnId,
1391 )).runtimeRequests.map((request) => request.id),
1392 retainedRuntimeRequestIds,
1393 );
1394 const replayed = yield* eventStore.read({ threadId }).pipe(Stream.runCollect);
1395 assert.equal(
1396 replayed.filter(
1397 (entry) =>
1398 entry.event.type === "runtime-request.updated" &&
1399 entry.event.payload.status === "cancelled",
1400 ).length,
1401 2,
1402 );
1403 const repeated = yield* ingestor.ingestNormalized(input);
1404 assert.isFalse(repeated.some((entry) => entry.event.type === "runtime-request.updated"));
1405 }),
1406 );
1407
1408 it.effect(
1409 "preserves an answer committed after terminal normalization reads a pending question",
1410 () =>
1411 Effect.gen(function* () {
1412 const eventSink = yield* EventSink.EventSinkV2;
1413 const eventStore = yield* EventStore.EventStoreV2;
1414 const projections = yield* ProjectionStore.ProjectionStoreV2;
1415 const idAllocator = yield* IdAllocator.IdAllocatorV2;
1416 const now = yield* DateTime.now;
1417 const threadEvent = yield* threadCreatedEvent(now);
1418 const threadId = threadEvent.threadId;
1419 const providerSessionId = yield* idAllocator.allocate.providerSession({
1420 providerInstanceId: modelSelection.instanceId,
1421 threadId,
1422 });
1423 const providerThreadId = idAllocator.derive.providerThread({
1424 driver: CODEX_DRIVER,
1425 nativeThreadId: `${threadId}:race`,
1426 });
1427 const providerTurnId = idAllocator.derive.providerTurn({
1428 driver: CODEX_DRIVER,
1429 nativeTurnId: `${threadId}:race`,
1430 });
1431 const nodeId = NodeId.make(`${threadId}:question`);
1432 const request: OrchestrationV2RuntimeRequest = {
1433 id: RuntimeRequestId.make(`${threadId}:question`),
1434 nodeId,
1435 providerTurnId,
1436 nativeRequestRef: null,
1437 kind: "user_input",
1438 status: "pending",
1439 responseCapability: { type: "live", providerSessionId },
1440 createdAt: now,
1441 resolvedAt: null,
1442 };
1443 const node: OrchestrationV2ExecutionNode = {
1444 id: nodeId,
1445 threadId,
1446 runId: null,
1447 parentNodeId: null,
1448 rootNodeId: nodeId,
1449 kind: "user_input_request",
1450 status: "waiting",
1451 countsForRun: false,
1452 providerThreadId,
1453 providerTurnId,
1454 nativeItemRef: null,
1455 runtimeRequestId: request.id,
1456 checkpointScopeId: null,
1457 startedAt: now,
1458 completedAt: null,
1459 };
1460 const item: OrchestrationV2TurnItem = {
1461 id: TurnItemId.make(`${threadId}:question`),
1462 threadId,
1463 runId: null,
1464 nodeId,
1465 providerThreadId,
1466 providerTurnId,
1467 nativeItemRef: null,
1468 parentItemId: null,
1469 ordinal: 1,
1470 status: "waiting",
1471 title: "Which option?",
1472 startedAt: now,
1473 completedAt: null,
1474 updatedAt: now,
1475 type: "user_input_request",
1476 requestId: request.id,
1477 questions: [],
1478 };
1479 const seedEvents: Array<OrchestrationV2DomainEvent> = [threadEvent];
1480 for (const payload of [
1481 { type: "runtime-request.updated" as const, payload: request },
1482 { type: "node.updated" as const, payload: node },
1483 { type: "turn-item.updated" as const, payload: item },
1484 ])
1485 seedEvents.push({
1486 id: yield* idAllocator.allocate.event({ threadId }),
1487 threadId,
1488 occurredAt: now,
1489 ...payload,
1490 });
1491 yield* eventSink.write({ events: seedEvents });
1492 const normalized = yield* Deferred.make<void>();
1493 const releaseTerminalWrite = yield* Deferred.make<void>();
1494 const gatedSink = EventSink.EventSinkV2.of({
1495 ...eventSink,
1496 write: (input) =>
1497 Deferred.succeed(normalized, undefined).pipe(
1498 Effect.andThen(Deferred.await(releaseTerminalWrite)),
1499 Effect.andThen(eventSink.write(input)),
1500 ),
1501 });
1502 const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2.pipe(
1503 Effect.provide(Layer.fresh(ProviderEventIngestor.layer)),
1504 Effect.provideService(EventSink.EventSinkV2, gatedSink),
1505 );
1506 const terminal = yield* ingestor
1507 .ingestNormalized({
1508 providerSessionId,
1509 providerInstanceId: modelSelection.instanceId,
1510 threadId,
1511 event: {
1512 type: "provider_turn.updated",
1513 driver: CODEX_DRIVER,
1514 providerTurn: {
1515 id: providerTurnId,
1516 providerThreadId,
1517 nodeId,
1518 runAttemptId: null,
1519 nativeTurnRef: null,
1520 ordinal: 1,
1521 status: "completed",
1522 startedAt: now,
1523 completedAt: now,
1524 },
1525 },
1526 })
1527 .pipe(Effect.forkScoped);
1528 yield* Deferred.await(normalized);
1529 const answers = { decision: "Use the existing workspace" };
1530 const responseEvents: Array<OrchestrationV2DomainEvent> = [];
1531 for (const payload of [
1532 {
1533 type: "runtime-request.updated" as const,
1534 payload: { ...request, status: "resolved" as const, answers, resolvedAt: now },
1535 },
1536 {
1537 type: "node.updated" as const,
1538 payload: { ...node, status: "completed" as const, completedAt: now },
1539 },
1540 {
1541 type: "turn-item.updated" as const,
1542 payload: { ...item, status: "completed" as const, completedAt: now },
1543 },
1544 ])
1545 responseEvents.push({
1546 id: yield* idAllocator.allocate.event({ threadId }),
1547 threadId,
1548 occurredAt: now,
1549 ...payload,
1550 });
1551 yield* eventSink.write({ events: responseEvents });
1552 yield* Deferred.succeed(releaseTerminalWrite, undefined);
1553 const committedTerminal = yield* Fiber.join(terminal);
1554 assert.deepEqual(
1555 committedTerminal.map((stored) => stored.event.type),
1556 ["provider-turn.updated"],
1557 );
1558 const projection = yield* projections.getThreadProjection(threadId);
1559 const answered = projection.runtimeRequests.find((entry) => entry.id === request.id)!;
1560 assert.equal(answered.status, "resolved");
1561 assert.deepEqual(answered.answers, answers);
1562 assert.equal(projection.nodes.find((entry) => entry.id === nodeId)!.status, "completed");
1563 assert.equal(
1564 projection.turnItems.find((entry) => entry.id === item.id)!.status,
1565 "completed",
1566 );
1567 const history = yield* eventStore.read({ threadId }).pipe(Stream.runCollect);
1568 assert.isFalse(
1569 history.some(
1570 (stored) =>
1571 stored.event.type === "runtime-request.updated" &&
1572 stored.event.payload.status === "cancelled",
1573 ),
1574 );
1575 }),
1576 );
1577
1578 it.effect("persists a failed provider terminal as one expected error item", () =>
1579 Effect.gen(function* () {
1580 const now = yield* DateTime.now;
1581 const retryStartedAt = DateTime.makeUnsafe(DateTime.toEpochMillis(now) - 5_000);
1582 const eventSink = yield* EventSink.EventSinkV2;
1583 const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
1584 const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
1585 const idAllocator = yield* IdAllocator.IdAllocatorV2;
1586 const threadEvent = yield* threadCreatedEvent(now);
1587 const providerSessionId = yield* idAllocator.allocate.providerSession({
1588 providerInstanceId: modelSelection.instanceId,
1589 threadId: threadEvent.threadId,
1590 });
1591 const providerThreadId = idAllocator.derive.providerThread({
1592 driver: CODEX_DRIVER,
1593 nativeThreadId: "native-thread-failed",
1594 });
1595 const providerTurnId = idAllocator.derive.providerTurn({
1596 driver: CODEX_DRIVER,
1597 nativeTurnId: "native-turn-failed",
1598 });
1599
1600 yield* eventSink.write({ events: [threadEvent] });
1601 const stored = yield* ingestor.ingestNormalized({
1602 providerSessionId,
1603 providerInstanceId: modelSelection.instanceId,
1604 threadId: threadEvent.threadId,
1605 event: {
1606 type: "turn.terminal",
1607 driver: CODEX_DRIVER,
1608 providerThreadId,
1609 providerTurnId,
1610 runOrdinal: 1,
1611 failureItemOrdinal: 102,
1612 status: "failed",
1613 failure: makeProviderFailure({
1614 message: "Invalid reasoning effort.",
1615 code: "invalid_request",
1616 class: "validation_error",
1617 }),
1618 retry: {
1619 attempt: 3,
1620 maxAttempts: 3,
1621 retryDelayMs: 2_000,
1622 },
1623 retryStartedAt,
1624 threadDisposition: "reusable",
1625 },
1626 });
1627
1628 const projection = yield* projectionStore.getThreadProjection(threadEvent.threadId);
1629 const errorItems = projection.visibleTurnItems.filter(
1630 (candidate) => candidate.item.type === "error",
1631 );
1632
1633 assert.equal(stored.length, 1);
1634 assert.equal(stored[0]?.event.type, "turn-item.updated");
1635 assert.equal(errorItems.length, 1);
1636 const errorItem = errorItems[0]?.item;
1637 assert.equal(errorItem?.type, "error");
1638 if (errorItem?.type !== "error") return;
1639 assert.equal(errorItem.failure.message, "Invalid reasoning effort.");
1640 assert.equal(errorItem.failure.code, "invalid_request");
1641 assert.deepEqual(errorItem.retry, {
1642 attempt: 3,
1643 maxAttempts: 3,
1644 retryDelayMs: 2_000,
1645 });
1646 const errorStartedAt = errorItem.startedAt;
1647 assert.ok(errorStartedAt);
1648 assert.equal(DateTime.toEpochMillis(errorStartedAt), DateTime.toEpochMillis(retryStartedAt));
1649 assert.equal(errorItem.providerThreadId, providerThreadId);
1650 assert.equal(errorItem.providerTurnId, providerTurnId);
1651 }),
1652 );
1653
1654 it.effect("stores tool image bytes only where a tool-output-image asset serves them", () =>
1655 Effect.gen(function* () {
1656 const now = yield* DateTime.now;
1657 const eventSink = yield* EventSink.EventSinkV2;
1658 const eventStore = yield* EventStore.EventStoreV2;
1659 const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
1660 const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
1661 const idAllocator = yield* IdAllocator.IdAllocatorV2;
1662 const threadEvent = yield* threadCreatedEvent(now);
1663 const providerSessionId = yield* idAllocator.allocate.providerSession({
1664 providerInstanceId: modelSelection.instanceId,
1665 threadId: threadEvent.threadId,
1666 });
1667 const readBase64 = Buffer.alloc(30_000, 7).toString("base64");
1668 const screenshotBase64 = Buffer.alloc(20_000, 9).toString("base64");
1669 const toolItem = (
1670 id: string,
1671 ordinal: number,
1672 toolName: string,
1673 output: unknown,
1674 ): OrchestrationV2TurnItem => ({
1675 id: TurnItemId.make(id),
1676 threadId: threadEvent.threadId,
1677 runId: null,
1678 nodeId: null,
1679 providerThreadId: null,
1680 providerTurnId: null,
1681 nativeItemRef: null,
1682 parentItemId: null,
1683 ordinal,
1684 status: "completed",
1685 title: toolName,
1686 startedAt: now,
1687 completedAt: now,
1688 updatedAt: now,
1689 type: "dynamic_tool",
1690 toolName,
1691 input: {},
1692 output,
1693 });
1694 const read = toolItem("turn-item:read-image", 1, "Read", {
1695 type: "image",
1696 file: { base64: readBase64, type: "image/png", originalSize: 30_000 },
1697 });
1698 const screenshot = toolItem("turn-item:screenshot", 2, "mcp__t3-code__device_screenshot", {
1699 content: [
1700 {
1701 type: "image",
1702 source: { type: "base64", media_type: "image/png", data: screenshotBase64 },
1703 },
1704 ],
1705 });
1706
1707 yield* eventSink.write({ events: [threadEvent] });
1708 for (const turnItem of [read, screenshot]) {
1709 yield* ingestor.ingestNormalized({
1710 providerSessionId,
1711 providerInstanceId: modelSelection.instanceId,
1712 threadId: threadEvent.threadId,
1713 event: { type: "turn_item.updated", driver: CODEX_DRIVER, turnItem },
1714 });
1715 }
1716
1717 const storedEvents = yield* eventStore
1718 .read({ threadId: threadEvent.threadId, eventType: "turn-item.updated" })
1719 .pipe(Stream.runCollect);
1720 const storedJson = JSON.stringify(Array.from(storedEvents, (stored) => stored.event));
1721 const projectedRead = yield* projectionStore.getTurnItem({
1722 threadId: threadEvent.threadId,
1723 itemId: read.id,
1724 });
1725 const projectedScreenshot = yield* projectionStore.getTurnItem({
1726 threadId: threadEvent.threadId,
1727 itemId: screenshot.id,
1728 });
1729
1730 assert.equal(storedJson.includes(readBase64), false);
1731 assert.equal(storedJson.includes(screenshotBase64), true);
1732 assert.deepEqual(projectedRead?.type === "dynamic_tool" ? projectedRead.output : null, {
1733 type: "image",
1734 file: { type: "image/png", originalSize: 30_000, sizeBytes: 30_000 },
1735 });
1736 assert.deepEqual(
1737 toolOutputImages(
1738 projectedScreenshot?.type === "dynamic_tool" ? projectedScreenshot.output : null,
1739 ),
1740 [{ mimeType: "image/png", data: screenshotBase64 }],
1741 );
1742 }),
1743 );
1744
1745 it.effect("keeps a steered message pending until the provider reports it read", () =>
1746 Effect.gen(function* () {
1747 const now = yield* DateTime.now;
1748 const eventSink = yield* EventSink.EventSinkV2;
1749 const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
1750 const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
1751 const idAllocator = yield* IdAllocator.IdAllocatorV2;
1752 const threadEvent = yield* threadCreatedEvent(now);
1753 const providerSessionId = yield* idAllocator.allocate.providerSession({
1754 providerInstanceId: modelSelection.instanceId,
1755 threadId: threadEvent.threadId,
1756 });
1757 const runId = RunId.make("run:steer-read");
1758 const messageId = MessageId.make("message:steer-read");
1759 const steer: OrchestrationV2TurnItem = {
1760 id: idAllocator.derive.userTurnItem({ messageId }),
1761 threadId: threadEvent.threadId,
1762 runId,
1763 nodeId: null,
1764 providerThreadId: null,
1765 providerTurnId: null,
1766 nativeItemRef: null,
1767 parentItemId: null,
1768 ordinal: 1,
1769 status: "completed",
1770 title: null,
1771 startedAt: now,
1772 completedAt: now,
1773 updatedAt: now,
1774 type: "user_message",
1775 createdBy: "user",
1776 creationSource: "web",
1777 messageId,
1778 inputIntent: "steer",
1779 text: "Steer behind a command",
1780 attachments: [],
1781 };
1782 yield* eventSink.write({ events: [threadEvent] });
1783 const ingest = (event: ProviderAdapter.ProviderAdapterV2Event) =>
1784 ingestor.ingestNormalized({
1785 providerSessionId,
1786 providerInstanceId: modelSelection.instanceId,
1787 threadId: threadEvent.threadId,
1788 event,
1789 });
1790 const projected = projectionStore.getTurnItem({
1791 threadId: threadEvent.threadId,
1792 itemId: steer.id,
1793 });
1794 const steerUpdated = (status: "pending" | "completed") =>
1795 ingest({
1796 type: "steer.updated",
1797 driver: CODEX_DRIVER,
1798 threadId: threadEvent.threadId,
1799 runId,
1800 messageId,
1801 status,
1802 });
1803
1804 yield* ingest({ type: "turn_item.updated", driver: CODEX_DRIVER, turnItem: steer });
1805 yield* steerUpdated("pending");
1806 assert.deepInclude(yield* projected, { status: "pending", completedAt: null });
1807 yield* steerUpdated("completed");
1808 assert.equal((yield* projected)?.status, "completed");
1809 }),
1810 );
1811
1812 it.effect("stores a bounded copy of a large file-change diff in both SQLite tables", () =>
1813 Effect.gen(function* () {
1814 const now = yield* DateTime.now;
1815 const eventSink = yield* EventSink.EventSinkV2;
1816 const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
1817 const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
1818 const idAllocator = yield* IdAllocator.IdAllocatorV2;
1819 const threadEvent = yield* threadCreatedEvent(now);
1820 const providerSessionId = yield* idAllocator.allocate.providerSession({
1821 providerInstanceId: modelSelection.instanceId,
1822 threadId: threadEvent.threadId,
1823 });
1824 const largeDiff = `@@ -0,0 +1 @@\n${"+generated line\n".repeat(150_000)}`;
1825 const fileChange = (
1826 id: string,
1827 ordinal: number,
1828 diffStr: string,
1829 ): OrchestrationV2TurnItem => ({
1830 id: TurnItemId.make(id),
1831 threadId: threadEvent.threadId,
1832 runId: null,
1833 nodeId: null,
1834 providerThreadId: null,
1835 providerTurnId: null,
1836 nativeItemRef: null,
1837 parentItemId: null,
1838 ordinal,
1839 status: "completed",
1840 title: null,
1841 startedAt: now,
1842 completedAt: now,
1843 updatedAt: now,
1844 type: "file_change",
1845 fileName: "src/generated.ts",
1846 diffStr,
1847 });
1848 const large = fileChange("turn-item:large-diff", 1, largeDiff);
1849 const small = fileChange("turn-item:small-diff", 2, "@@ -1 +1 @@\n-old\n+new\n");
1850
1851 yield* eventSink.write({ events: [threadEvent] });
1852 for (const turnItem of [large, small]) {
1853 yield* ingestor.ingestNormalized({
1854 providerSessionId,
1855 providerInstanceId: modelSelection.instanceId,
1856 threadId: threadEvent.threadId,
1857 event: { type: "turn_item.updated", driver: CODEX_DRIVER, turnItem },
1858 });
1859 }
1860
1861 const sql = yield* SqlClient.SqlClient;
1862 const storedBytes = yield* sql<{ readonly events: number; readonly projection: number }>`
1863 SELECT
1864 (SELECT SUM(LENGTH(payload_json)) FROM orchestration_events
1865 WHERE stream_id = ${threadEvent.threadId} AND event_type = 'turn-item.updated') AS events,
1866 (SELECT SUM(LENGTH(payload_json)) FROM orchestration_v2_projection_turn_items
1867 WHERE thread_id = ${threadEvent.threadId}) AS projection
1868 `;
1869 const projectedLarge = yield* projectionStore.getTurnItem({
1870 threadId: threadEvent.threadId,
1871 itemId: large.id,
1872 });
1873 const projectedSmall = yield* projectionStore.getTurnItem({
1874 threadId: threadEvent.threadId,
1875 itemId: small.id,
1876 });
1877
1878 assert.isAbove(largeDiff.length, 2_000_000);
1879 assert.isBelow(storedBytes[0]!.events, 100_000);
1880 assert.isBelow(storedBytes[0]!.projection, 100_000);
1881 assert.isTrue(projectedLarge?.type === "file_change");
1882 if (projectedLarge?.type !== "file_change") return;
1883 assert.equal(projectedLarge.fileName, "src/generated.ts");
1884 assert.isBelow(projectedLarge.diffStr?.length ?? 0, 40_000);
1885 assert.isTrue(largeDiff.startsWith(projectedLarge.diffStr?.split("\n…")[0] ?? "-"));
1886 assert.deepEqual(projectedSmall, small);
1887 }),
1888 );
1889
1890 it.effect("routes provider-owned child artifacts to their child app thread", () =>
1891 Effect.gen(function* () {
1892 const now = yield* DateTime.now;
1893 const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
1894 const idAllocator = yield* IdAllocator.IdAllocatorV2;
1895 const rootEvent = yield* threadCreatedEvent(now);
1896 if (rootEvent.type !== "thread.created") {
1897 throw new Error("Expected a thread.created fixture event");
1898 }
1899 const childThreadId = idAllocator.derive.threadFromProviderThread({
1900 driver: CODEX_DRIVER,
1901 nativeThreadId: "native-subagent-thread",
1902 });
1903 const childRootNodeId = NodeId.make("node:subagent-root");
1904 const childThread: OrchestrationV2AppThread = {
1905 ...rootEvent.payload,
1906 id: childThreadId,
1907 title: "inspect package",
1908 activeProviderThreadId: null,
1909 lineage: {
1910 parentThreadId: rootEvent.threadId,
1911 relationshipToParent: "subagent",
1912 rootThreadId: rootEvent.threadId,
1913 },
1914 forkedFrom: {
1915 type: "node",
1916 nodeId: NodeId.make("node:parent-subagent"),
1917 },
1918 };
1919 const providerSessionId = yield* idAllocator.allocate.providerSession({
1920 providerInstanceId: modelSelection.instanceId,
1921 threadId: rootEvent.threadId,
1922 });
1923
1924 const failedSpans: Array<string> = [];
1925 const tracer = Tracer.make({
1926 span(options) {
1927 const span = new Tracer.NativeSpan(options);
1928 const end = span.end.bind(span);
1929 span.end = (time, exit) => {
1930 end(time, exit);
1931 if (Exit.isFailure(exit)) failedSpans.push(span.name);
1932 };
1933 return span;
1934 },
1935 });
1936 const threadEvents = yield* ingestor
1937 .normalize({
1938 providerSessionId,
1939 providerInstanceId: modelSelection.instanceId,
1940 threadId: rootEvent.threadId,
1941 event: {
1942 type: "app_thread.created",
1943 driver: CODEX_DRIVER,
1944 appThread: childThread,
1945 },
1946 })
1947 .pipe(Effect.withTracer(tracer));
1948 // A new child's existence check expects a miss; a failed span there reads as an error in Tempo.
1949 assert.deepEqual(failedSpans, []);
1950 const messageEvents = yield* ingestor.normalize({
1951 providerSessionId,
1952 providerInstanceId: modelSelection.instanceId,
1953 threadId: rootEvent.threadId,
1954 event: {
1955 type: "message.updated",
1956 driver: CODEX_DRIVER,
1957 message: {
1958 createdBy: "agent",
1959 creationSource: "provider",
1960 id: MessageId.make("message:subagent-response"),
1961 threadId: childThreadId,
1962 runId: null,
1963 nodeId: childRootNodeId,
1964 role: "assistant",
1965 text: "Subagent result",
1966 attachments: [],
1967 streaming: false,
1968 createdAt: now,
1969 updatedAt: now,
1970 },
1971 },
1972 });
1973
1974 assert.equal(threadEvents[0]?.type, "thread.created");
1975 assert.equal(threadEvents[0]?.threadId, childThreadId);
1976 assert.equal(messageEvents[0]?.type, "message.updated");
1977 assert.equal(messageEvents[0]?.threadId, childThreadId);
1978 }),
1979 );
1980
1981 it.effect.each([
1982 { parentMoved: false, failure: null, temporaryBranch: "t3/abcd1234" },
1983 { parentMoved: true, failure: null, temporaryBranch: "t3/abcd1234" },
1984 { parentMoved: false, failure: "write", temporaryBranch: "t3/abcd1234" },
1985 { parentMoved: false, failure: "git", temporaryBranch: "t3/abcd1234" },
1986 { parentMoved: false, failure: null, temporaryBranch: "my-prefix/abcd1234" },
1987 ])(
1988 "rechecks a pending native child's branch after the rename scan: %j",
1989 ({ parentMoved, failure, temporaryBranch }) =>
1990 Effect.gen(function* () {
1991 const { worktreePath, command } = yield* makeSharedWorktree;
1992 if (temporaryBranch !== "t3/abcd1234")
1993 yield* command(worktreePath, ["branch", "-m", temporaryBranch]);
1994 const git = yield* GitVcsDriver.GitVcsDriver;
1995 const now = yield* DateTime.now;
1996 const eventSink = yield* EventSink.EventSinkV2;
1997 const projections = yield* ProjectionStore.ProjectionStoreV2;
1998 const ids = yield* IdAllocator.IdAllocatorV2;
1999 const rootEvent = yield* threadCreatedEvent(now);
2000 if (rootEvent.type !== "thread.created") throw new Error("Expected thread fixture");
2001 const parent = {
2002 ...rootEvent.payload,
2003 branch: temporaryBranch,
2004 worktreePath,
2005 };
2006 yield* eventSink.write({ events: [{ ...rootEvent, payload: parent }] });
2007 const childId = ids.derive.threadFromProviderThread({
2008 driver: CODEX_DRIVER,
2009 nativeThreadId: `child-created-during-rename:${parentMoved}:${failure}:${temporaryBranch}`,
2010 });
2011 const beforeCommit = yield* Deferred.make<
2012 void,
2013 ProviderEventIngestor.ProviderEventIngestorV2Error
2014 >();
2015 const allowCommit = yield* Deferred.make<void>();
2016 let correctionWrites = 0;
2017 const gatedSink = EventSink.EventSinkV2.of({
2018 ...eventSink,
2019 write: (input) => {
2020 if (
2021 input.events.some(
2022 (event) => event.type === "thread.metadata-updated" && event.threadId === childId,
2023 )
2024 ) {
2025 correctionWrites++;
2026 if (failure === "write" && correctionWrites === 1)
2027 return Effect.fail(new EventSink.EventSinkWriteError({ eventCount: 1 }));
2028 }
2029 return input.events.some(
2030 (event) => event.type === "thread.created" && event.threadId === childId,
2031 )
2032 ? Deferred.succeed(beforeCommit, undefined).pipe(
2033 Effect.andThen(Deferred.await(allowCommit)),
2034 Effect.andThen(eventSink.write(input)),
2035 )
2036 : eventSink.write(input);
2037 },
2038 });
2039 const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2.pipe(
2040 Effect.provide(Layer.fresh(ProviderEventIngestor.layer)),
2041 Effect.provideService(EventSink.EventSinkV2, gatedSink),
2042 Effect.provideService(GitVcsDriver.GitVcsDriver, {
2043 ...git,
2044 resolveBranchName: (cwd, refName) =>
2045 failure === "git"
2046 ? Effect.fail(
2047 new GitCommandError({
2048 operation: "ingestor-test",
2049 cwd,
2050 command: "rev-parse",
2051 detail: "Read failed",
2052 }),
2053 )
2054 : git.resolveBranchName(cwd, refName),
2055 }),
2056 );
2057 const providerSessionId = yield* ids.allocate.providerSession({
2058 providerInstanceId: modelSelection.instanceId,
2059 threadId: parent.id,
2060 });
2061 const creationInput = {
2062 providerSessionId,
2063 providerInstanceId: modelSelection.instanceId,
2064 threadId: parent.id,
2065 event: {
2066 type: "app_thread.created",
2067 driver: CODEX_DRIVER,
2068 appThread: {
2069 ...parent,
2070 id: childId,
2071 lineage: {
2072 parentThreadId: parent.id,
2073 rootThreadId: parent.id,
2074 relationshipToParent: "subagent",
2075 },
2076 },
2077 },
2078 } satisfies ProviderEventIngestor.ProviderEventIngestInput;
2079 const creation = yield* ingestor.ingestNormalized(creationInput).pipe(
2080 Effect.tapCause((cause) => Deferred.failCause(beforeCommit, cause)),
2081 Effect.forkScoped,
2082 );
2083 yield* Deferred.await(beforeCommit);
2084 yield* command(worktreePath, ["branch", "-m", "generated-branch"]);
2085 yield* eventSink.write({
2086 events: [
2087 {
2088 ...rootEvent,
2089 id: yield* ids.allocate.event({ threadId: parent.id }),
2090 type: "thread.metadata-updated",
2091 payload: {
2092 ...parent,
2093 branch: parentMoved ? "user/checkout" : "generated-branch",
2094 worktreePath: parentMoved ? `${worktreePath}-moved` : worktreePath,
2095 },
2096 },
2097 ],
2098 });
2099 const snapshot = yield* projections.getShellSnapshot();
2100 assert.isFalse(
2101 [...snapshot.threads, ...snapshot.archivedThreads].some(
2102 (thread) =>
2103 thread.worktreePath === parent.worktreePath && thread.branch === parent.branch,
2104 ),
2105 );
2106 yield* Deferred.succeed(allowCommit, undefined);
2107 const stored = yield* Fiber.join(creation);
2108 assert.equal(stored[0]?.event.type, "thread.created");
2109 const child = yield* projections.getThread(childId);
2110 assert.equal(child.branch, failure === null ? "generated-branch" : parent.branch);
2111 assert.equal(child.worktreePath, parent.worktreePath);
2112 assert.equal(correctionWrites, failure === "git" ? 0 : 1);
2113 assert.deepEqual(yield* ingestor.ingestNormalized(creationInput), []);
2114 }),
2115 );
2116
2117 // regression: #20079 the renamed-branch correction must not rewrite a child that needs none
2118 it.effect.each(["explicit", "detached"])(
2119 "preserves a native child's branch when the checkout needs no correction: %s",
2120 (scenario) =>
2121 Effect.gen(function* () {
2122 const { worktreePath, command } = yield* makeSharedWorktree;
2123 const git = yield* GitVcsDriver.GitVcsDriver;
2124 const branch = scenario === "explicit" ? "child-feature" : "t3/abcd1234";
2125 if (scenario === "explicit") yield* command(worktreePath, ["branch", "-m", branch]);
2126 if (scenario === "detached") yield* command(worktreePath, ["checkout", "--detach"]);
2127 const now = yield* DateTime.now;
2128 const sink = yield* EventSink.EventSinkV2;
2129 const projections = yield* ProjectionStore.ProjectionStoreV2;
2130 const ids = yield* IdAllocator.IdAllocatorV2;
2131 const root = yield* threadCreatedEvent(now);
2132 if (root.type !== "thread.created") throw new Error("Expected thread fixture");
2133 yield* sink.write({
2134 events: [
2135 { ...root, payload: { ...root.payload, branch: "parent-feature", worktreePath } },
2136 ],
2137 });
2138 const childId = ids.derive.threadFromProviderThread({
2139 driver: CODEX_DRIVER,
2140 nativeThreadId: `no-correction:${scenario}`,
2141 });
2142 let gitReads = 0;
2143 const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2.pipe(
2144 Effect.provide(Layer.fresh(ProviderEventIngestor.layer)),
2145 Effect.provideService(GitVcsDriver.GitVcsDriver, {
2146 ...git,
2147 resolveBranchName: (cwd, refName) => {
2148 gitReads++;
2149 return git.resolveBranchName(cwd, refName);
2150 },
2151 }),
2152 );
2153 const stored = yield* ingestor.ingestNormalized({
2154 providerSessionId: yield* ids.allocate.providerSession({
2155 providerInstanceId: modelSelection.instanceId,
2156 threadId: root.threadId,
2157 }),
2158 providerInstanceId: modelSelection.instanceId,
2159 threadId: root.threadId,
2160 event: {
2161 type: "app_thread.created",
2162 driver: CODEX_DRIVER,
2163 appThread: {
2164 ...root.payload,
2165 id: childId,
2166 branch,
2167 worktreePath,
2168 lineage: {
2169 parentThreadId: root.threadId,
2170 rootThreadId: root.threadId,
2171 relationshipToParent: "subagent",
2172 },
2173 },
2174 },
2175 });
2176 assert.equal((yield* projections.getThread(childId)).branch, branch);
2177 assert.equal(stored.length, 1);
2178 assert.equal(gitReads, 1);
2179 }),
2180 );
2181
2182 it.effect("updates provider child metadata without recreating its saved thread", () =>
2183 Effect.gen(function* () {
2184 const now = yield* DateTime.now;
2185 const eventSink = yield* EventSink.EventSinkV2;
2186 const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
2187 const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
2188 const idAllocator = yield* IdAllocator.IdAllocatorV2;
2189 const rootEvent = yield* threadCreatedEvent(now);
2190 if (rootEvent.type !== "thread.created") throw new Error("Expected thread fixture");
2191 const childThreadId = idAllocator.derive.threadFromProviderThread({
2192 driver: CODEX_DRIVER,
2193 nativeThreadId: "workflow-member-metadata",
2194 });
2195 const providerSessionId = yield* idAllocator.allocate.providerSession({
2196 providerInstanceId: modelSelection.instanceId,
2197 threadId: rootEvent.threadId,
2198 });
2199 const offered = {
2200 ...rootEvent.payload,
2201 id: childThreadId,
2202 title: "Inspect checkout",
2203 activeProviderThreadId: null,
2204 lineage: {
2205 parentThreadId: rootEvent.threadId,
2206 relationshipToParent: "subagent" as const,
2207 rootThreadId: rootEvent.threadId,
2208 },
2209 };
2210 const ingest = (appThread: OrchestrationV2AppThread) =>
2211 ingestor.ingestNormalized({
2212 providerSessionId,
2213 providerInstanceId: modelSelection.instanceId,
2214 threadId: rootEvent.threadId,
2215 event: { type: "app_thread.created", driver: CODEX_DRIVER, appThread },
2216 });
2217 yield* eventSink.write({ events: [rootEvent] });
2218 const created = yield* Effect.all([ingest(offered), ingest(offered)], {
2219 concurrency: "unbounded",
2220 });
2221 assert.deepEqual(
2222 created.flat().map((stored) => stored.event.type),
2223 ["thread.created"],
2224 );
2225 const saved = {
2226 ...offered,
2227 archivedAt: now,
2228 deletedAt: now,
2229 activeProviderThreadId: rootEvent.payload.activeProviderThreadId,
2230 branch: "user/checkout",
2231 };
2232 yield* eventSink.write({
2233 events: [
2234 {
2235 ...rootEvent,
2236 id: yield* idAllocator.allocate.event({ threadId: childThreadId }),
2237 threadId: childThreadId,
2238 type: "thread.deleted",
2239 payload: saved,
2240 },
2241 ],
2242 });
2243 for (const metadata of [
2244 { title: "Review checkout", modelSelection },
2245 { title: "Review checkout", modelSelection: { ...modelSelection, model: "gpt-6.1-sol" } },
2246 ]) {
2247 const updated = yield* ingest({ ...offered, ...metadata });
2248 assert.deepEqual(
2249 updated.map((stored) => stored.event.type),
2250 ["thread.metadata-updated"],
2251 );
2252 const thread = yield* projectionStore.getThread(childThreadId);
2253 assert.deepEqual(thread, { ...saved, ...metadata, updatedAt: thread.updatedAt });
2254 }
2255 assert.deepEqual(
2256 yield* ingest({
2257 ...offered,
2258 title: "Review checkout",
2259 modelSelection: { ...modelSelection, model: "gpt-6.1-sol" },
2260 }),
2261 [],
2262 );
2263 }),
2264 );
2265
2266 it.effect(
2267 "recovers a missing Claude child once and preserves saved metadata on later recovery",
2268 () =>
2269 Effect.gen(function* () {
2270 const now = yield* DateTime.now;
2271 const eventSink = yield* EventSink.EventSinkV2;
2272 const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
2273 const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
2274 const idAllocator = yield* IdAllocator.IdAllocatorV2;
2275 const rootEvent = yield* threadCreatedEvent(now);
2276 if (rootEvent.type !== "thread.created") throw new Error("Expected thread fixture");
2277 const driver = ProviderDriverKind.make("claude");
2278 const providerInstanceId = ProviderInstanceId.make("claude");
2279 const childThreadId = idAllocator.derive.threadFromProviderThread({
2280 driver,
2281 nativeThreadId: "recovered-claude-child",
2282 });
2283 const providerSessionId = yield* idAllocator.allocate.providerSession({
2284 providerInstanceId,
2285 threadId: rootEvent.threadId,
2286 });
2287 const recovered: OrchestrationV2AppThread = {
2288 ...rootEvent.payload,
2289 id: childThreadId,
2290 title: "Recovered subagent",
2291 providerInstanceId,
2292 modelSelection: { instanceId: providerInstanceId, model: "claude-sonnet-4-6" },
2293 activeProviderThreadId: null,
2294 createdBy: "agent",
2295 creationSource: "provider",
2296 lineage: {
2297 parentThreadId: rootEvent.threadId,
2298 relationshipToParent: "subagent",
2299 rootThreadId: rootEvent.threadId,
2300 },
2301 };
2302 const ingest = (appThread: OrchestrationV2AppThread, createOnly: boolean) =>
2303 ingestor.ingestNormalized({
2304 providerSessionId,
2305 providerInstanceId,
2306 threadId: rootEvent.threadId,
2307 event: { type: "app_thread.created", driver, appThread, createOnly },
2308 });
2309 yield* eventSink.write({ events: [rootEvent] });
2310 const created = yield* Effect.all([ingest(recovered, true), ingest(recovered, true)], {
2311 concurrency: "unbounded",
2312 });
2313 assert.deepEqual(
2314 created.flat().map((stored) => stored.event.type),
2315 ["thread.created"],
2316 );
2317 assert.deepEqual(yield* projectionStore.getThread(childThreadId), recovered);
2318
2319 yield* ingest(
2320 {
2321 ...recovered,
2322 title: "Review authentication",
2323 modelSelection: { instanceId: providerInstanceId, model: "claude-opus-4-6" },
2324 },
2325 false,
2326 );
2327 const saved = yield* projectionStore.getThread(childThreadId);
2328 assert.equal(saved.title, "Review authentication");
2329 assert.equal(saved.modelSelection.model, "claude-opus-4-6");
2330 assert.deepEqual(yield* ingest(recovered, true), []);
2331 assert.deepEqual(yield* projectionStore.getThread(childThreadId), saved);
2332 }),
2333 );
2334
2335 it.effect("moves a native subagent's thread to its later model through a database lock", () =>
2336 Effect.gen(function* () {
2337 const now = yield* DateTime.now;
2338 const eventSink = yield* EventSink.EventSinkV2;
2339 const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
2340 const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
2341 const idAllocator = yield* IdAllocator.IdAllocatorV2;
2342 const rootEvent = yield* threadCreatedEvent(now);
2343 if (rootEvent.type !== "thread.created") {
2344 throw new Error("Expected a thread.created fixture event");
2345 }
2346 const childThreadId = idAllocator.derive.threadFromProviderThread({
2347 driver: CODEX_DRIVER,
2348 nativeThreadId: "native-late-model-subagent",
2349 });
2350 const providerSessionId = yield* idAllocator.allocate.providerSession({
2351 providerInstanceId: modelSelection.instanceId,
2352 threadId: rootEvent.threadId,
2353 });
2354 const ingest = (event: ProviderEventIngestor.ProviderEventIngestInput["event"]) =>
2355 ingestor.ingestNormalized({
2356 providerSessionId,
2357 providerInstanceId: modelSelection.instanceId,
2358 threadId: rootEvent.threadId,
2359 event,
2360 });
2361 yield* eventSink.write({ events: [rootEvent] });
2362 yield* ingest({
2363 type: "app_thread.created",
2364 driver: CODEX_DRIVER,
2365 appThread: {
2366 ...rootEvent.payload,
2367 id: childThreadId,
2368 title: "review design",
2369 modelSelection: {
2370 ...modelSelection,
2371 options: [{ id: "reasoningEffort", value: "xhigh" }],
2372 },
2373 activeProviderThreadId: null,
2374 lineage: {
2375 parentThreadId: rootEvent.threadId,
2376 relationshipToParent: "subagent",
2377 rootThreadId: rootEvent.threadId,
2378 },
2379 },
2380 });
2381 const subagentUpdated = {
2382 type: "subagent.updated",
2383 driver: CODEX_DRIVER,
2384 subagent: {
2385 id: NodeId.make("node:late-model-subagent"),
2386 threadId: rootEvent.threadId,
2387 runId: null,
2388 parentNodeId: NodeId.make("node:root"),
2389 origin: "provider_native",
2390 createdBy: "agent",
2391 driver: CODEX_DRIVER,
2392 providerInstanceId: modelSelection.instanceId,
2393 providerThreadId: null,
2394 childThreadId,
2395 nativeTaskRef: null,
2396 prompt: "Review the design",
2397 title: "review design",
2398 model: "gpt-6.1-sol",
2399 status: "running",
2400 result: null,
2401 startedAt: now,
2402 completedAt: null,
2403 updatedAt: now,
2404 },
2405 } satisfies ProviderEventIngestor.ProviderEventIngestInput["event"];
2406
2407 const first = yield* ingest(subagentUpdated);
2408 const repeated = yield* ingest(subagentUpdated);
2409 const childThread = yield* projectionStore.getThread(childThreadId);
2410
2411 assert.deepEqual(
2412 first.map((stored) => [stored.event.type, stored.event.threadId]),
2413 [
2414 ["subagent.updated", rootEvent.threadId],
2415 ["thread.model-selection-updated", childThreadId],
2416 ],
2417 );
2418 assert.deepEqual(
2419 repeated.map((stored) => stored.event.type),
2420 ["subagent.updated"],
2421 );
2422 assert.deepEqual(childThread.modelSelection, {
2423 instanceId: modelSelection.instanceId,
2424 model: "gpt-6.1-sol",
2425 });
2426
2427 const reportedSelection = {
2428 instanceId: modelSelection.instanceId,
2429 model: "gpt-6.1-sol",
2430 options: [{ id: "reasoningEffort", value: "low" }],
2431 };
2432 const reported = yield* ingest({
2433 ...subagentUpdated,
2434 subagent: { ...subagentUpdated.subagent, modelSelection: reportedSelection },
2435 });
2436 assert.deepEqual(
2437 reported.map((stored) => stored.event.type),
2438 ["subagent.updated", "thread.model-selection-updated"],
2439 );
2440 assert.deepEqual(
2441 (yield* projectionStore.getThread(childThreadId)).modelSelection,
2442 reportedSelection,
2443 );
2444
2445 const recovered = yield* ingest(subagentUpdated);
2446 assert.deepEqual(
2447 recovered.map((stored) => stored.event.type),
2448 ["subagent.updated"],
2449 );
2450 assert.deepEqual(
2451 (yield* projectionStore.getThread(childThreadId)).modelSelection,
2452 reportedSelection,
2453 );
2454 }).pipe(
2455 // Fresh: the shared test layer has already built the ingestor this replaces.
2456 Effect.provide(
2457 Layer.fresh(
2458 layerFailingWrites([lockTimeout, lockTimeout], (events) =>
2459 events.some((event) => event.type === "thread.model-selection-updated"),
2460 ),
2461 ),
2462 ),
2463 ),
2464 );
2465 });
2466
2467 const ingestCompletedProviderTurn = (
2468 guard: Pick<
2469 Parameters<ProviderEventIngestor.ProviderEventIngestorV2Shape["ingestNormalized"]>[0],
2470 "writeIfRunCurrent" | "writeIfProviderThreadOwner"
2471 >,
2472 ) =>
2473 Effect.gen(function* () {
2474 const now = yield* DateTime.now;
2475 const eventSink = yield* EventSink.EventSinkV2;
2476 const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
2477 const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
2478 const idAllocator = yield* IdAllocator.IdAllocatorV2;
2479 const threadEvent = yield* threadCreatedEvent(now);
2480 yield* eventSink.write({ events: [threadEvent] });
2481 const providerTurnId = ProviderTurnId.make("provider-turn:locked");
2482 const exit = yield* ingestor
2483 .ingestNormalized({
2484 providerSessionId: yield* idAllocator.allocate.providerSession({
2485 providerInstanceId: modelSelection.instanceId,
2486 threadId: threadEvent.threadId,
2487 }),
2488 providerInstanceId: modelSelection.instanceId,
2489 threadId: threadEvent.threadId,
2490 event: {
2491 type: "provider_turn.updated",
2492 driver: CODEX_DRIVER,
2493 providerTurn: {
2494 id: providerTurnId,
2495 providerThreadId: ProviderThreadId.make("provider-thread:locked"),
2496 nodeId: NodeId.make("node:locked-turn"),
2497 runAttemptId: null,
2498 nativeTurnRef: null,
2499 ordinal: 1,
2500 status: "completed",
2501 startedAt: now,
2502 completedAt: now,
2503 },
2504 },
2505 ...guard,
2506 })
2507 .pipe(Effect.exit);
2508 const projection = yield* projectionStore.getThreadProjection(threadEvent.threadId);
2509 return { exit, providerTurnId, persisted: projection.providerTurns.map((turn) => turn.id) };
2510 });
2511
2512 // Neither guard matches a stored run, so a guarded write correctly commits nothing.
2513 const runId = RunId.make("run:locked");
2514 const activeAttemptId = RunAttemptId.make("attempt:locked");
2515 it.effect.each([
2516 { route: "write", guard: {} },
2517 {
2518 route: "writeIfRunCurrent",
2519 guard: { writeIfRunCurrent: { runId, activeAttemptId, expectedStatus: "running" } },
2520 },
2521 {
2522 route: "writeIfProviderThreadOwner",
2523 guard: {
2524 writeIfProviderThreadOwner: {
2525 providerThreadId: ProviderThreadId.make("provider-thread:locked"),
2526 runId,
2527 activeAttemptId,
2528 expectedLastRunOrdinal: 1,
2529 },
2530 },
2531 },
2532 ] as const)("a provider event $route outlasts a brief database lock", ({ route, guard }) =>
2533 Effect.gen(function* () {
2534 const { exit, providerTurnId, persisted } = yield* ingestCompletedProviderTurn(guard);
2535 assert.isTrue(Exit.isSuccess(exit));
2536 assert.deepEqual(persisted, route === "write" ? [providerTurnId] : []);
2537 }).pipe(Effect.provide(layerFailingWrites([lockTimeout, lockTimeout]))),
2538 );
2539
2540 // The write after the queued errors would succeed, so failing proves the retries stopped.
2541 it.effect.each([
2542 { failure: "stays locked", errors: [lockTimeout, lockTimeout, lockTimeout] },
2543 { failure: "fails for another reason", errors: [constraintFailure] },
2544 ])("fails a provider event write that $failure", ({ errors }) =>
2545 Effect.gen(function* () {
2546 const { exit, persisted } = yield* ingestCompletedProviderTurn({});
2547 assert.instanceOf(
2548 Exit.isFailure(exit) ? Cause.squash(exit.cause) : undefined,
2549 ProviderEventIngestor.ProviderEventPublishError,
2550 );
2551 assert.isEmpty(persisted);
2552 }).pipe(Effect.provide(layerFailingWrites(errors))),
2553 );