@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

main
ts 4,680 lines 177 KB
Raw
1 import type * as NodeWorkerThreads from "node:worker_threads";
2 import * as NodeServices from "@effect/platform-node/NodeServices";
3 import { assert, it } from "@effect/vitest";
4 import {
5 CheckpointId,
6 CheckpointRef,
7 CheckpointScopeId,
8 CommandId,
9 ContextTransferId,
10 EventId,
11 MessageId,
12 type ModelSelection,
13 NodeId,
14 type OrchestrationV2AppThread,
15 type OrchestrationV2DomainEvent,
16 type OrchestrationV2Run,
17 type OrchestrationV2ThreadProjection,
18 type OrchestrationV2TurnItem,
19 ProjectId,
20 ProviderDriverKind,
21 ProviderInstanceId,
22 ProviderSessionId,
23 ProviderThreadId,
24 RunAttemptId,
25 RunId,
26 ThreadId,
27 TurnItemId,
28 } from "@t3tools/contracts";
29 import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient";
30 import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
31 import * as ConfigProvider from "effect/ConfigProvider";
32 import * as Queue from "effect/Queue";
33 import * as Arr from "effect/Array";
34 import * as Cause from "effect/Cause";
35 import * as Clock from "effect/Clock";
36 import * as DateTime from "effect/DateTime";
37 import * as Exit from "effect/Exit";
38 import * as Deferred from "effect/Deferred";
39 import * as Duration from "effect/Duration";
40 import * as Effect from "effect/Effect";
41 import * as Fiber from "effect/Fiber";
42 import * as FileSystem from "effect/FileSystem";
43 import * as Layer from "effect/Layer";
44 import * as Option from "effect/Option";
45 import * as Path from "effect/Path";
46 import * as Ref from "effect/Ref";
47 import * as Scheduler from "effect/Scheduler";
48 import * as Schema from "effect/Schema";
49 import * as Stream from "effect/Stream";
50 import * as TestClock from "effect/testing/TestClock";
51 import * as Tracer from "effect/Tracer";
52 import * as SqlClient from "effect/sql/SqlClient";
53 import * as Statement from "effect/sql/Statement";
54
55 import { vi } from "vite-plus/test";
56
57 import * as ServerConfig from "../config.ts";
58 import * as DatabaseStorage from "../persistence/DatabaseStorage.ts";
59 import * as ServerSettings from "../serverSettings.ts";
60 import { LIVE_STREAM_MAX_ITEMS, LiveStreamBufferError } from "./LiveStreamBudget.ts";
61 import * as SqlitePersistence from "../persistence/Sqlite.ts";
62 import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
63 import * as CommandReceiptStore from "./CommandReceiptStore.ts";
64 import * as EffectOutbox from "./EffectOutbox.ts";
65 import * as EffectWorker from "./EffectWorker.ts";
66 import * as EventSink from "./EventSink.ts";
67 import * as ThreadViewers from "./ThreadViewers.ts";
68 import * as EventStore from "./EventStore.ts";
69 import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
70 import * as ProjectionMaintenance from "./ProjectionMaintenance.ts";
71 import * as ProjectionReplay from "./testkit/ProjectionReplay.ts";
72 import * as ProjectionStore from "./ProjectionStore.ts";
73 import * as ProjectStore from "./ProjectStore.ts";
74 import * as ProviderRuntimeRecovery from "./ProviderRuntimeRecoveryService.ts";
75 import * as TurnItemPositionStore from "./TurnItemPositionStore.ts";
76
77 const workerStatements = vi.hoisted(
78 (): Array<{ readonly readOnly: boolean; readonly sql: string }> => [],
79 );
80
81 vi.mock("node:worker_threads", async (importOriginal) => {
82 const actual = await importOriginal<typeof NodeWorkerThreads>();
83 class RecordedWorker extends actual.Worker {
84 readonly #readOnly: boolean;
85 constructor(...args: ConstructorParameters<typeof actual.Worker>) {
86 super(...args);
87 this.#readOnly = args[1]?.workerData?.readOnly === true;
88 }
89 override postMessage(...args: Parameters<NodeWorkerThreads.Worker["postMessage"]>) {
90 const [message] = args;
91 if (typeof message === "object" && message !== null && typeof message.sql === "string") {
92 workerStatements.push({ readOnly: this.#readOnly, sql: message.sql });
93 }
94 return super.postMessage(...args);
95 }
96 }
97 return { ...actual, Worker: RecordedWorker };
98 });
99
100 const isLiveStreamBufferError = Schema.is(LiveStreamBufferError);
101 const encodeJson = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown));
102
103 const layerDatabase = SqlitePersistence.layerMemory;
104 const layerEventStoreProvided = EventStore.layer.pipe(Layer.provideMerge(layerDatabase));
105 const layerProjectionStoreProvided = ProjectionStore.layer.pipe(Layer.provideMerge(layerDatabase));
106 const layerStoresProvided = Layer.mergeAll(
107 layerDatabase,
108 layerEventStoreProvided,
109 layerProjectionStoreProvided,
110 );
111 const layerEventSinkProvided = EventSink.layer.pipe(Layer.provide(layerStoresProvided));
112 const layerEffectOutboxProvided = EffectOutbox.layer.pipe(Layer.provide(layerDatabase));
113 const layerCommandReceiptStoreProvided = CommandReceiptStore.layer.pipe(
114 Layer.provide(layerDatabase),
115 );
116 const layerIsolatedOutbox = Layer.fresh(EffectOutbox.layer.pipe(Layer.provideMerge(layerDatabase)));
117 const layerProjectionMaintenanceProvided = ProjectionMaintenance.layer.pipe(
118 Layer.provide(layerDatabase),
119 );
120 const layerProjectionReplayProvided = ProjectionReplay.layer.pipe(
121 Layer.provide(layerStoresProvided),
122 );
123 const layerTest = Layer.mergeAll(
124 layerStoresProvided,
125 layerEventSinkProvided,
126 layerEffectOutboxProvided,
127 layerCommandReceiptStoreProvided,
128 IdAllocator.layer,
129 layerProjectionMaintenanceProvided,
130 layerProjectionReplayProvided,
131 );
132
133 const providerInstanceId = ProviderInstanceId.make("codex");
134 const providerDriver = ProviderDriverKind.make("codex");
135 const modelSelection = {
136 instanceId: providerInstanceId,
137 model: "gpt-5.4",
138 } satisfies ModelSelection;
139
140 function makeThread(threadId: ThreadId, now: DateTime.Utc): OrchestrationV2AppThread {
141 return {
142 createdBy: "user",
143 creationSource: "web",
144 id: threadId,
145 projectId: ProjectId.make(`project:${threadId}`),
146 title: `Thread ${threadId}`,
147 providerInstanceId,
148 modelSelection,
149 runtimeMode: "full-access",
150 interactionMode: "default",
151 branch: null,
152 worktreePath: null,
153 activeProviderThreadId: null,
154 lineage: {
155 parentThreadId: null,
156 relationshipToParent: null,
157 rootThreadId: threadId,
158 },
159 forkedFrom: null,
160 createdAt: now,
161 updatedAt: now,
162 archivedAt: null,
163 settledOverride: null,
164 settledAt: null,
165 lastVisitedAt: null,
166 deletedAt: null,
167 };
168 }
169
170 function threadCreatedEvent(input: {
171 readonly id: string;
172 readonly thread: OrchestrationV2AppThread;
173 readonly now: DateTime.Utc;
174 }): OrchestrationV2DomainEvent {
175 return {
176 id: EventId.make(input.id),
177 type: "thread.created",
178 threadId: input.thread.id,
179 providerInstanceId,
180 occurredAt: input.now,
181 payload: input.thread,
182 };
183 }
184
185 const seedCompaction = Effect.fnUntraced(function* () {
186 const sql = yield* SqlClient.SqlClient;
187 yield* sql`INSERT INTO orchestration_v2_legacy_imports (
188 thread_id, source_updated_at, shell_imported_at, transcript_imported_at, imported_message_count
189 ) VALUES ('imported', '2026-10-17', '2026-10-17', '2026-10-17', 0)`;
190 yield* sql`WITH RECURSIVE history(n) AS (
191 SELECT 1 UNION ALL SELECT n + 1 FROM history WHERE n < 151
192 ) INSERT INTO orchestration_events (
193 event_id, aggregate_kind, stream_id, stream_version, event_type,
194 occurred_at, actor_kind, payload_json, metadata_json, application_event_version
195 ) SELECT 'event-' || n, 'thread', 'thread', n,
196 CASE WHEN n IN (1, 51, 101) THEN 'message.updated' ELSE 'thread.visited' END,
197 '2026-10-17', 'server', '{"id":"message"}', '{}', 2 FROM history`;
198 // A pass a day ago marked the whole log; projections cover it.
199 yield* sql`INSERT INTO orchestration_v2_projection_metadata (
200 projection_name, schema_version, last_sequence, updated_at
201 ) VALUES
202 ('event-log-prune-mark', 2, 151, '1969-12-30T00:00:00.000Z'),
203 ('thread-projections', 2, 151, '1969-12-30T00:00:00.000Z')
204 ON CONFLICT(projection_name) DO UPDATE SET last_sequence = excluded.last_sequence`;
205 yield* sql`WITH RECURSIVE history(n) AS (
206 SELECT 1 UNION ALL SELECT n + 1 FROM history WHERE n < 100
207 ) INSERT INTO orchestration_command_receipts (
208 rowid, command_id, aggregate_kind, aggregate_id, accepted_at, result_sequence, status, command_type
209 ) SELECT n * 2, 'command-' || n, 'thread', 'imported', '2026-10-17', 0, 'accepted', 'legacy' FROM history`;
210 });
211
212 it.effect("discovers on the reader worker and deletes on the writer", () =>
213 Effect.gen(function* () {
214 const fs = yield* FileSystem.FileSystem;
215 const path = yield* Path.Path;
216 const filename = path.join(yield* fs.makeTempDirectoryScoped(), "fixture.sqlite");
217 yield* Effect.gen(function* () {
218 const sql = yield* SqlClient.SqlClient;
219 yield* seedCompaction();
220 // Establish the reader before recording statements.
221 yield* sql`SELECT 1`.pipe(NodeSqliteClient.readOnly);
222 workerStatements.length = 0;
223 const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2;
224 const result = yield* maintenance.pruneEventStore();
225 assert.equal(result.deletedEventCount, 150);
226 assert.equal(result.deletedReceiptCount, 100);
227 const writer = workerStatements.filter((statement) => !statement.readOnly);
228 // Only the in-transaction check for rows appended during the pass reads on the writer.
229 assert.deepEqual(
230 writer.filter(({ sql }) => /\bSELECT\b/.test(sql) && !sql.includes("AS prompt_sequence")),
231 [],
232 );
233 assert(workerStatements.some(({ readOnly, sql }) => readOnly && /\bSELECT\b/.test(sql)));
234 assert(writer.some(({ sql }) => sql.includes("DELETE FROM orchestration_events")));
235 assert(writer.some(({ sql }) => sql.includes("DELETE FROM orchestration_command_receipts")));
236 }).pipe(
237 Effect.provide(
238 ProjectionMaintenance.layer.pipe(
239 Layer.provideMerge(SqlitePersistence.layerFromPath(filename)),
240 ),
241 ),
242 );
243 }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
244 );
245
246 it.effect("paces scheduled compaction while preserving retention state and fixed scan bounds", () =>
247 Effect.gen(function* () {
248 const sql = yield* SqlClient.SqlClient;
249 const clock = yield* Clock.Clock;
250 yield* seedCompaction();
251 const completed = yield* Deferred.make<void>();
252 const pauseStarted = yield* Deferred.make<void>();
253 let activeNanos = 0n;
254 let eventPages = 0;
255 const pauses: number[] = [];
256 const compactionSpans: Tracer.Span[] = [];
257 const tracer = Tracer.make({
258 span(options) {
259 const span = new Tracer.NativeSpan(options);
260 if (span.name === "ProjectionMaintenance.pruneEventStore") compactionSpans.push(span);
261 const end = span.end.bind(span);
262 span.end = (time, exit) => {
263 end(time, exit);
264 const query = span.attributes.get("db.query.text");
265 if (typeof query !== "string" || !query.includes("LIMIT")) return;
266 if (query.includes("FROM orchestration_events AS event")) {
267 eventPages++;
268 } else if (!query.includes("FROM orchestration_command_receipts AS receipt")) return;
269 // Model a page exceeding the active budget, without wall-clock sleeps.
270 activeNanos += 30_000_000n;
271 };
272 return span;
273 },
274 });
275 yield* Effect.gen(function* () {
276 yield* DatabaseStorage.DatabaseStorage;
277 yield* TestClock.adjust("1 minute");
278 yield* Effect.raceFirst(Deferred.await(pauseStarted), Deferred.await(completed));
279 yield* TestClock.adjust("269 millis");
280 assert.equal(eventPages, 1);
281 assert.deepEqual(pauses, [270]);
282 assert.deepEqual(
283 compactionSpans.map((span) => span.status._tag),
284 ["Started"],
285 );
286 // Foreground commits must not extend the compactor's initial scan bounds.
287 yield* sql`INSERT INTO orchestration_events (
288 event_id, aggregate_kind, stream_id, stream_version, event_type,
289 occurred_at, actor_kind, payload_json, metadata_json, application_event_version
290 ) VALUES ('after-start', 'thread', 'thread', 152, 'thread.visited', '2026-10-17', 'server', '{}', '{}', 2)`;
291 yield* sql`INSERT INTO orchestration_command_receipts (
292 command_id, aggregate_kind, aggregate_id, accepted_at, result_sequence, status, command_type
293 ) VALUES ('after-start', 'thread', 'imported', '2026-10-17', 0, 'accepted', 'legacy')`;
294 yield* TestClock.adjust("10 seconds");
295 yield* Deferred.await(completed);
296 }).pipe(
297 Effect.provide(
298 DatabaseStorage.layer.pipe(
299 Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "compaction-" })),
300 Layer.provide(NodeServices.layer),
301 ),
302 ),
303 Effect.withTracer(tracer),
304 Effect.provideService(Clock.Clock, {
305 ...clock,
306 monotonicTimeNanos: Effect.sync(() => activeNanos),
307 sleep: (duration) => {
308 const millis = Duration.toMillis(duration);
309 if (millis < 60_000) {
310 pauses.push(millis);
311 Deferred.doneUnsafe(pauseStarted, Effect.void);
312 } else if (millis === 6 * 60 * 60_000) {
313 Deferred.doneUnsafe(completed, Effect.void);
314 }
315 return clock.sleep(duration);
316 },
317 }),
318 );
319 assert.deepEqual(pauses, Array(8).fill(270));
320 assert.deepEqual(
321 compactionSpans.map((span) => span.status._tag),
322 ["Ended"],
323 );
324 // 151 stopped being the stream head when 152 committed mid-pass; 152 is past the bound.
325 assert.deepEqual(yield* sql`SELECT sequence FROM orchestration_events ORDER BY sequence`, [
326 { sequence: 152 },
327 ]);
328 assert.deepEqual(yield* sql`SELECT command_id FROM orchestration_command_receipts`, [
329 { command_id: "after-start" },
330 ]);
331 }).pipe(Effect.scoped, Effect.provide(layerTest)),
332 );
333
334 it.effect(
335 "commits an ordered provider batch atomically and publishes only after the outer commit",
336 () =>
337 Effect.gen(function* () {
338 const sink = yield* EventSink.EventSinkV2;
339 const projections = yield* ProjectionStore.ProjectionStoreV2;
340 const store = yield* EventStore.EventStoreV2;
341 const now = yield* DateTime.now;
342 const thread = makeThread(ThreadId.make("thread:single-stream-batch"), now);
343 yield* sink.write({ events: [threadCreatedEvent({ id: "batch-created", thread, now })] });
344 const message = (id: string): OrchestrationV2DomainEvent => ({
345 id: EventId.make(id),
346 type: "message.updated",
347 threadId: thread.id,
348 occurredAt: now,
349 payload: {
350 id: MessageId.make("batch-message"),
351 threadId: thread.id,
352 runId: null,
353 nodeId: null,
354 createdBy: "agent",
355 creationSource: "provider",
356 role: "assistant",
357 text: id,
358 streaming: true,
359 attachments: [],
360 createdAt: now,
361 updatedAt: now,
362 },
363 });
364 const highWater = yield* sink.latestSequence();
365 const callerError = { _tag: "CallerError" } as const;
366 assert.strictEqual(
367 yield* sink.withProviderBatch(Effect.fail(callerError)).pipe(Effect.flip),
368 callerError,
369 );
370 const failed = yield* sink
371 .withProviderBatch(
372 Effect.gen(function* () {
373 yield* sink.write({ events: [message("batch-duplicate")] });
374 yield* sink.write({ events: [message("batch-duplicate")] });
375 }),
376 )
377 .pipe(Effect.exit);
378 assert.isTrue(Exit.isFailure(failed));
379 assert.equal(yield* sink.latestSequence(), highWater);
380 assert.isEmpty((yield* projections.getThreadRecords(thread.id, ["messages"])).messages);
381
382 const firstPublished = yield* Deferred.make<void>();
383 const writesApplied = yield* Deferred.make<void>();
384 const release = yield* Deferred.make<void>();
385 const live = yield* sink.stream({ afterSequence: highWater }).pipe(
386 Stream.tap(() => Deferred.succeed(firstPublished, undefined)),
387 Stream.take(2),
388 Stream.runCollect,
389 Effect.forkScoped({ startImmediately: true }),
390 );
391 const batch = yield* sink
392 .withProviderBatch(
393 Effect.gen(function* () {
394 yield* sink.write({ events: [message("batch-first")] });
395 yield* sink.write({ events: [message("batch-second")] });
396 assert.equal(
397 (yield* projections.getThreadRecords(thread.id, ["messages"])).messages[0]?.text,
398 "batch-second",
399 );
400 yield* Deferred.succeed(writesApplied, undefined);
401 yield* Deferred.await(release);
402 }),
403 )
404 .pipe(Effect.forkScoped({ startImmediately: true }));
405 yield* Deferred.await(writesApplied);
406 yield* TestClock.adjust(0);
407 assert.isFalse(yield* Deferred.isDone(firstPublished));
408 assert.isUndefined(yield* Effect.sync(() => batch.pollUnsafe()));
409 yield* Deferred.succeed(release, undefined);
410 yield* Fiber.join(batch);
411 const published = yield* Fiber.join(live);
412 assert.deepEqual(
413 published.map((stored) => stored.event.id),
414 ["batch-first", "batch-second"],
415 );
416 assert.deepEqual(
417 yield* store.read({ afterSequence: highWater }).pipe(Stream.runCollect),
418 published,
419 );
420 }).pipe(Effect.provide(Layer.fresh(layerTest))),
421 );
422
423 function nodeComparisonEvent(thread: OrchestrationV2AppThread, now: DateTime.Utc) {
424 const nodeId = NodeId.make(`node:${thread.id}`);
425 return {
426 id: EventId.make(`node-cache:${thread.id}`),
427 type: "node.updated",
428 threadId: thread.id,
429 occurredAt: now,
430 payload: {
431 id: nodeId,
432 threadId: thread.id,
433 runId: null,
434 parentNodeId: null,
435 rootNodeId: nodeId,
436 kind: "assistant_message",
437 status: "running",
438 countsForRun: false,
439 providerThreadId: null,
440 providerTurnId: null,
441 nativeItemRef: null,
442 runtimeRequestId: null,
443 checkpointScopeId: null,
444 startedAt: now,
445 completedAt: null,
446 },
447 } satisfies OrchestrationV2DomainEvent;
448 }
449
450 it.effect("commits other threads' writes ahead of a thread's queued provider snapshots", () =>
451 Effect.gen(function* () {
452 const sink = yield* EventSink.EventSinkV2;
453 const now = yield* DateTime.now;
454 const [busy, independent] = ["busy", "independent"].map((id) =>
455 makeThread(ThreadId.make(id), now),
456 );
457 yield* sink.write({
458 events: [busy!, independent!].map((thread) =>
459 threadCreatedEvent({ id: `created:${thread.id}`, thread, now }),
460 ),
461 });
462 const highWater = yield* sink.latestSequence();
463 const busyNode = nodeComparisonEvent(busy!, now);
464 const snapshot = {
465 type: "node.updated" as const,
466 driver: providerDriver,
467 node: busyNode.payload,
468 };
469 const started = yield* Deferred.make<void>();
470 const release = yield* Deferred.make<void>();
471 const pulled = yield* Deferred.make<void>();
472 let writes = 0;
473 let pulls = 0;
474 const input = yield* Queue.unbounded<typeof snapshot>();
475 const provider = yield* sink
476 .consumeProviderEvents(
477 Stream.fromQueue(input).pipe(
478 Stream.take(3),
479 Stream.tap(() => (++pulls === 3 ? Deferred.succeed(pulled, undefined) : Effect.void)),
480 ),
481 () =>
482 Effect.gen(function* () {
483 const id = EventId.make(`busy-provider-${++writes}`);
484 yield* sink.write({ events: [{ ...busyNode, id }] });
485 if (writes > 1) return;
486 yield* Deferred.succeed(started, undefined);
487 yield* Deferred.await(release);
488 }),
489 Effect.succeed(false),
490 Effect.succeed(true),
491 )
492 .pipe(Effect.forkScoped({ startImmediately: true }));
493 yield* Queue.offer(input, snapshot);
494 yield* Deferred.await(started);
495 yield* Queue.offerAll(
496 input,
497 [2, 3].map((n) => ({
498 ...snapshot,
499 node: { ...busyNode.payload, id: NodeId.make(`node:busy:${n}`) },
500 })),
501 );
502 yield* Deferred.await(pulled);
503 yield* TestClock.adjust(0);
504 const writeTo = (event: OrchestrationV2DomainEvent, id: string) =>
505 sink
506 .write({ events: [{ ...event, id: EventId.make(id) }] })
507 .pipe(Effect.forkScoped({ startImmediately: true }));
508 const sameThread = yield* writeTo(busyNode, "busy-command");
509 const otherThread = yield* writeTo(nodeComparisonEvent(independent!, now), "independent");
510 const childOfBusy = yield* writeTo(
511 {
512 id: EventId.make("unused"),
513 type: "provider-thread.updated",
514 threadId: independent!.id,
515 driver: providerDriver,
516 providerInstanceId,
517 occurredAt: now,
518 payload: {
519 id: ProviderThreadId.make("provider-thread:busy-child"),
520 driver: providerDriver,
521 providerInstanceId,
522 providerSessionId: null,
523 appThreadId: busy!.id,
524 ownerNodeId: null,
525 nativeThreadRef: null,
526 nativeConversationHeadRef: null,
527 status: "idle",
528 firstRunOrdinal: 1,
529 lastRunOrdinal: 1,
530 handoffIds: [],
531 forkedFrom: null,
532 createdAt: now,
533 updatedAt: now,
534 },
535 },
536 "busy-child",
537 );
538 yield* TestClock.adjust(0);
539 yield* Deferred.succeed(release, undefined);
540 yield* Fiber.join(provider);
541 yield* Fiber.join(sameThread);
542 yield* Fiber.join(otherThread);
543 yield* Fiber.join(childOfBusy);
544 const store = yield* EventStore.EventStoreV2;
545 const events = yield* store.read({ afterSequence: highWater }).pipe(Stream.runCollect);
546 assert.deepEqual(
547 events.map((stored) => stored.event.id),
548 [
549 "busy-provider-1",
550 "independent",
551 "busy-provider-2",
552 "busy-provider-3",
553 "busy-command",
554 "busy-child",
555 ],
556 );
557 }).pipe(Effect.scoped, Effect.provide(Layer.fresh(layerTest))),
558 );
559
560 it.effect(
561 "reuses node comparisons and writes thread bookkeeping once per provider transaction",
562 () =>
563 Effect.gen(function* () {
564 const sink = yield* EventSink.EventSinkV2;
565 const projections = yield* ProjectionStore.ProjectionStoreV2;
566 const sql = yield* SqlClient.SqlClient;
567 const now = yield* DateTime.now;
568 const thread = makeThread(ThreadId.make("node-comparison-cache"), now);
569 yield* sink.write({
570 events: [threadCreatedEvent({ id: "node-cache-created", thread, now })],
571 });
572 const event = nodeComparisonEvent(thread, now);
573 const statements = { nodeReads: 0, threadTouches: 0, highWaterWrites: 0 };
574 const observe: Statement.Transformer = (statement) => {
575 const text = statement.compile()[0];
576 if (
577 text.includes("SELECT payload_json FROM orchestration_v2_projection_nodes WHERE node_id")
578 )
579 statements.nodeReads++;
580 if (text.includes("UPDATE orchestration_v2_projection_threads")) statements.threadTouches++;
581 if (text.includes("INSERT INTO orchestration_v2_projection_metadata"))
582 statements.highWaterWrites++;
583 return Effect.succeed(statement);
584 };
585 const lastOccurredAt = DateTime.add(now, { minutes: 3 });
586 yield* sink
587 .withProviderBatch(
588 Effect.gen(function* () {
589 for (let n = 0; n < 21; n++)
590 assert.isFalse(yield* projections.isNodeStored(event.payload));
591 assert.equal(statements.nodeReads, 1);
592 yield* sink.write({ events: [event] });
593 assert.isTrue(yield* projections.isNodeStored(event.payload));
594 for (const minutes of [1, 2, 3]) {
595 yield* sink.write({
596 events: [
597 {
598 ...event,
599 id: EventId.make(`node-cache-touch:${minutes}`),
600 occurredAt: DateTime.add(now, { minutes }),
601 },
602 ],
603 });
604 }
605 assert.deepEqual([statements.threadTouches, statements.highWaterWrites], [0, 0]);
606 }),
607 )
608 .pipe(Effect.provideService(Statement.CurrentTransformer, observe));
609 assert.deepEqual([statements.threadTouches, statements.highWaterWrites], [1, 1]);
610 assert.deepEqual(
611 DateTime.toEpochMillis((yield* projections.getThread(thread.id)).updatedAt),
612 DateTime.toEpochMillis(lastOccurredAt),
613 );
614 assert.deepEqual(
615 yield* sql`SELECT last_sequence FROM orchestration_v2_projection_metadata
616 WHERE projection_name = 'thread-projections'`,
617 [{ last_sequence: yield* sink.latestSequence() }],
618 );
619 const completed = { ...event.payload, status: "completed" as const, completedAt: now };
620 yield* sink.write({
621 events: [{ ...event, id: EventId.make("node-cache-completed"), payload: completed }],
622 });
623 yield* sink.withProviderBatch(
624 Effect.gen(function* () {
625 assert.isFalse(yield* projections.isNodeStored(event.payload));
626 assert.isTrue(yield* projections.isNodeStored(completed));
627 }),
628 );
629 }).pipe(Effect.scoped, Effect.provide(Layer.fresh(layerTest))),
630 );
631
632 it.effect.each(
633 (["none", "duplicate", "after-write"] as const).flatMap((failure) =>
634 [1, 10].map((streams) => ({ failure, streams })),
635 ),
636 )(
637 "shares the writer and isolates failure=$failure across $streams independent consumers",
638 ({ failure, streams }) => {
639 const rollback = failure !== "none";
640 const failedIndex = failure === "after-write" ? 0 : streams - 1;
641 const jobError = { _tag: "InvalidUpdate" } as const;
642 const writes: number[] = [];
643 const tracer = Tracer.make({
644 span(options) {
645 const span = new Tracer.NativeSpan(options);
646 const end = span.end.bind(span);
647 span.end = (time, exit) => {
648 end(time, exit);
649 if (span.name === "orchestrationV2.EventSink.providerBatch" && Exit.isSuccess(exit)) {
650 writes.push(Number(span.attributes.get("orchestration_v2.batch_writes")));
651 }
652 };
653 return span;
654 },
655 });
656 return Effect.gen(function* () {
657 const sink = yield* EventSink.EventSinkV2;
658 const projections = yield* ProjectionStore.ProjectionStoreV2;
659 const now = yield* DateTime.now;
660 const threads = Array.from({ length: streams }, (_, index) =>
661 makeThread(ThreadId.make(`shared:${index}`), now),
662 );
663 yield* sink.write({
664 events: threads.map((thread, index) =>
665 threadCreatedEvent({ id: `created:${index}`, thread, now }),
666 ),
667 });
668 const node = nodeComparisonEvent(threads[0]!, now);
669 yield* sink.write({ events: [node] });
670 const highWater = yield* sink.latestSequence();
671 const store = yield* EventStore.EventStoreV2;
672 const publications = yield* Queue.unbounded<EventId>();
673 yield* sink.stream({ afterSequence: highWater }).pipe(
674 Stream.runForEach((stored) => Queue.offer(publications, stored.event.id)),
675 Effect.forkScoped({ startImmediately: true }),
676 );
677 const pulled = yield* Deferred.make<void>();
678 let arrivals = 0;
679 const inputs = yield* Effect.forEach(threads, () =>
680 Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event, Cause.Done>(),
681 );
682 const consumers = yield* Effect.forEach(inputs, (input, index) =>
683 sink
684 .consumeProviderEvents(
685 Stream.fromQueue(input).pipe(
686 Stream.tap(() =>
687 ++arrivals === streams ? Deferred.succeed(pulled, undefined) : Effect.void,
688 ),
689 ),
690 (raw) =>
691 raw.type === "message.updated"
692 ? Effect.gen(function* () {
693 if (failure === "after-write" && index !== failedIndex) {
694 assert.isTrue(yield* projections.isNodeStored(node.payload));
695 }
696 yield* sink.write({
697 events: [
698 {
699 id: EventId.make(
700 failure === "duplicate" && index === failedIndex
701 ? "created:0"
702 : `shared-event:${index}`,
703 ),
704 threadId: raw.message.threadId,
705 occurredAt: DateTime.add(now, { minutes: index + 1 }),
706 type: "message.updated",
707 payload: raw.message,
708 },
709 ...(failure === "after-write" && index === failedIndex
710 ? [
711 {
712 ...node,
713 id: EventId.make("rolled-back-node"),
714 occurredAt: DateTime.add(now, { hours: 1 }),
715 payload: {
716 ...node.payload,
717 status: "completed" as const,
718 completedAt: now,
719 },
720 },
721 ]
722 : []),
723 ],
724 });
725 if (failure === "after-write" && index === failedIndex)
726 return yield* Effect.fail(jobError);
727 })
728 : Effect.void,
729 Effect.succeed(false),
730 Effect.succeed(true),
731 )
732 .pipe(Effect.exit, Effect.forkScoped({ startImmediately: true })),
733 );
734 for (let index = 0; index < threads.length; index++) {
735 yield* Queue.offer(inputs[index]!, {
736 type: "message.updated",
737 driver: ProviderDriverKind.make("codex"),
738 message: {
739 id: MessageId.make(`message:${index}`),
740 threadId: threads[index]!.id,
741 runId: null,
742 nodeId: null,
743 createdBy: "agent",
744 creationSource: "provider",
745 role: "assistant",
746 text: `stream:${index}`,
747 streaming: true,
748 attachments: [],
749 createdAt: now,
750 updatedAt: now,
751 },
752 });
753 }
754 yield* Deferred.await(pulled);
755 yield* Effect.forEach(inputs, Queue.end, { discard: true });
756 const exits = yield* Effect.forEach(consumers, Fiber.join);
757 if (rollback) {
758 const exit = exits[failedIndex]!;
759 assert.isTrue(Exit.isFailure(exit));
760 if (failure === "after-write" && Exit.isFailure(exit))
761 assert.strictEqual(Cause.squash(exit.cause), jobError);
762 }
763 yield* TestClock.adjust(0);
764 const replay = yield* store.read({ afterSequence: highWater }).pipe(Stream.runCollect);
765 assert.deepEqual(
766 replay.length === 0 ? [] : yield* Queue.takeN(publications, replay.length),
767 replay.map((stored) => stored.event.id),
768 );
769 assert.equal(yield* Queue.size(publications), 0);
770 assert.deepEqual(writes, rollback && streams === 1 ? [] : [streams - (rollback ? 1 : 0)]);
771 assert.equal(yield* sink.latestSequence(), highWater + streams - (rollback ? 1 : 0));
772 const sql = yield* SqlClient.SqlClient;
773 assert.deepEqual(
774 yield* sql`SELECT last_sequence FROM orchestration_v2_projection_metadata
775 WHERE projection_name = 'thread-projections'`,
776 [{ last_sequence: yield* sink.latestSequence() }],
777 );
778 for (let index = 0; index < threads.length; index++) {
779 const failed = rollback && index === failedIndex;
780 assert.lengthOf(
781 (yield* projections.getThreadRecords(threads[index]!.id, ["messages"])).messages,
782 failed ? 0 : 1,
783 );
784 assert.equal(
785 DateTime.toEpochMillis((yield* projections.getThread(threads[index]!.id)).updatedAt),
786 DateTime.toEpochMillis(failed ? now : DateTime.add(now, { minutes: index + 1 })),
787 );
788 }
789 }).pipe(
790 Effect.scoped,
791 Effect.provide(Layer.fresh(layerTest)),
792 Effect.withTracer(tracer),
793 Effect.withTracerEnabled(true),
794 Effect.provideService(
795 ConfigProvider.ConfigProvider,
796 ConfigProvider.fromUnknown({ T3CODE_PROVIDER_COMMIT_BATCHING: true }),
797 ),
798 );
799 },
800 );
801
802 it.effect.each([false, true])("groups shutdown commits with rollback=%s", (fail) => {
803 let commits = 0;
804 const tracer = Tracer.make({
805 span(options) {
806 const span = new Tracer.NativeSpan(options);
807 const end = span.end.bind(span);
808 span.end = (time, exit) => {
809 end(time, exit);
810 if (span.name === "COMMIT") commits++;
811 };
812 return span;
813 },
814 });
815 return Effect.gen(function* () {
816 const sink = yield* EventSink.EventSinkV2;
817 const events = yield* EventStore.EventStoreV2;
818 const projections = yield* ProjectionStore.ProjectionStoreV2;
819 const sql = yield* SqlClient.SqlClient;
820 const outbox = yield* EffectOutbox.EffectOutboxV2;
821 const now = yield* DateTime.now;
822 const threads = Array.from({ length: 65 }, (_, i) =>
823 ThreadId.make(`shutdown-group:${String(i).padStart(2, "0")}`),
824 );
825 const sharedSession = {
826 id: ProviderSessionId.make("shutdown-shared"),
827 driver: providerDriver,
828 providerInstanceId,
829 status: "ready" as const,
830 cwd: process.cwd(),
831 model: modelSelection.model,
832 capabilities: CodexProviderCapabilitiesV2,
833 createdAt: now,
834 updatedAt: now,
835 lastError: null,
836 };
837 yield* sink.write({
838 events: threads.flatMap((threadId, i): OrchestrationV2DomainEvent[] => [
839 threadCreatedEvent({ id: `shutdown-create:${i}`, thread: makeThread(threadId, now), now }),
840 ...(i < 2
841 ? [
842 {
843 id: EventId.make(`shutdown-session:${i}`),
844 type: "provider-session.attached" as const,
845 threadId,
846 occurredAt: now,
847 payload: sharedSession,
848 },
849 ]
850 : []),
851 {
852 id: EventId.make(`shutdown-provider:${i}`),
853 type: "provider-thread.updated",
854 threadId,
855 occurredAt: now,
856 payload: {
857 id: ProviderThreadId.make(`shutdown-provider:${i}`),
858 driver: providerDriver,
859 providerInstanceId,
860 providerSessionId: i < 2 ? sharedSession.id : null,
861 appThreadId: threadId,
862 ownerNodeId: null,
863 nativeThreadRef: null,
864 nativeConversationHeadRef: null,
865 status: i === 1 || i === 32 ? "idle" : "active",
866 firstRunOrdinal: null,
867 lastRunOrdinal: null,
868 handoffIds: [],
869 forkedFrom: null,
870 createdAt: now,
871 updatedAt: now,
872 },
873 },
874 ]),
875 });
876 yield* outbox.enqueue(
877 threads.map((threadId, i) => ({
878 id: `shutdown-effect:${i}`,
879 commandId: CommandId.make(`shutdown-command:${i}`),
880 threadId,
881 request: { type: "provider-turn.start" as const, runId: RunId.make(`shutdown-run:${i}`) },
882 })),
883 );
884 let eventlessNotifications = 0;
885 let armed = fail;
886 const recovery = yield* ProviderRuntimeRecovery.make.pipe(
887 Effect.provide(ServerSettings.layerTest()),
888 Effect.provideService(EffectOutbox.EffectOutboxV2, {
889 ...outbox,
890 signalCancellations: (ids) =>
891 Effect.gen(function* () {
892 assert.isTrue(Option.isNone(yield* Effect.serviceOption(sql.transactionService)));
893 eventlessNotifications += ids.length;
894 yield* outbox.signalCancellations(ids);
895 }),
896 }),
897 Effect.provideService(
898 EffectWorker.OrchestrationEffectWorkerV2,
899 EffectWorker.OrchestrationEffectWorkerV2.of({
900 awaitWork: Effect.void,
901 runRecoveryOnce: Effect.succeed(false),
902 runOnce: () => Effect.succeed(false),
903 nextClaimableAt: () => Effect.succeedNone,
904 drain: () => Effect.succeed(0),
905 }),
906 ),
907 Effect.provideService(EventSink.EventSinkV2, {
908 ...sink,
909 commitCommand: (input) =>
910 armed && input.threadId === threads[33]
911 ? Effect.die("injected shutdown failure")
912 : sink.commitCommand(input),
913 }),
914 );
915 commits = 0;
916 const result = yield* recovery.reconcile("shutdown").pipe(Effect.exit);
917 assert.equal(Exit.isFailure(result), fail);
918 assert.equal(commits, fail ? 1 : 3);
919 const receipts =
920 yield* sql`SELECT aggregate_id FROM orchestration_command_receipts WHERE command_type = 'provider-runtime.reconcile' ORDER BY aggregate_id`;
921 assert.deepEqual(
922 receipts.map((r) => r.aggregate_id),
923 threads.slice(0, fail ? 32 : 65).filter((_, i) => i !== 1 && i !== 32),
924 );
925 assert.deepEqual(
926 yield* projections.getRecoveryThreadIds("runtime"),
927 fail ? threads.slice(32) : [],
928 );
929 assert.equal(eventlessNotifications, fail ? 1 : 2);
930 for (const [i] of threads.entries()) {
931 const effect = Option.getOrThrow(yield* outbox.get(`shutdown-effect:${i}`));
932 assert.equal(effect.status, !fail || i < 32 ? "cancelled" : "pending");
933 }
934 assert.equal(
935 (yield* events.read({ afterSequence: 0 }).pipe(Stream.runCollect)).filter(
936 (e) =>
937 e.event.type === "provider-session.updated" && e.event.payload.id === sharedSession.id,
938 ).length,
939 1,
940 );
941 armed = false;
942 yield* recovery.reconcile("shutdown");
943 assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), []);
944 const highWater = yield* sink.latestSequence();
945 yield* recovery.reconcile("shutdown");
946 assert.equal(yield* sink.latestSequence(), highWater);
947 }).pipe(
948 Effect.provide(Layer.fresh(layerTest)),
949 Effect.withTracer(tracer),
950 Effect.withTracerEnabled(true),
951 Effect.provideService(Statement.SpanPropagationEnabled, true),
952 );
953 });
954
955 it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
956 it.effect("projects oversized tool bodies once before replay and live RPC retention", () =>
957 Effect.gen(function* () {
958 const sink = yield* EventSink.EventSinkV2;
959 const store = yield* EventStore.EventStoreV2;
960 const projections = yield* ProjectionStore.ProjectionStoreV2;
961 const now = yield* DateTime.now;
962 const thread = makeThread(ThreadId.make("thread:large-stream-body"), now);
963 const output = {
964 content: [{ type: "text", text: "first line\n" + "x".repeat(9 * 1024 * 1024) }],
965 };
966 const tool: OrchestrationV2TurnItem = {
967 id: TurnItemId.make("tool:large-stream-body"),
968 type: "dynamic_tool",
969 threadId: thread.id,
970 runId: null,
971 nodeId: null,
972 providerThreadId: null,
973 providerTurnId: null,
974 nativeItemRef: null,
975 parentItemId: null,
976 ordinal: 1,
977 status: "running",
978 title: "Large tool",
979 toolName: "mcp__large_tool",
980 input: { query: "details" },
981 output,
982 startedAt: now,
983 completedAt: null,
984 updatedAt: now,
985 };
986 const [created] = yield* sink.write({
987 events: [
988 threadCreatedEvent({ id: "event:large-stream-body:created", thread, now }),
989 {
990 id: EventId.make("event:large-stream-body:replay"),
991 type: "turn-item.updated",
992 threadId: thread.id,
993 occurredAt: now,
994 payload: tool,
995 },
996 ],
997 });
998 const subscribe = () =>
999 Stream.toPull(
1000 sink.stream({
1001 threadId: thread.id,
1002 afterSequence: created!.sequence,
1003 bounded: true,
1004 }),
1005 );
1006 const pull = yield* subscribe();
1007 const other = yield* subscribe();
1008 const replay = yield* pull;
1009 assert.lengthOf(replay, 1);
1010 assert.lengthOf(yield* other, 1);
1011 assert.equal(replay[0]!.event.type, "turn-item.updated");
1012 assert.isBelow(Buffer.byteLength(yield* encodeJson(replay)), 4_000);
1013 const stringify = yield* Effect.acquireRelease(
1014 Effect.sync(() => vi.spyOn(JSON, "stringify")),
1015 (spy) => Effect.sync(() => spy.mockRestore()),
1016 );
1017 const [completed] = yield* sink.write({
1018 events: [
1019 {
1020 id: EventId.make("event:large-stream-body:live"),
1021 type: "turn-item.updated",
1022 threadId: thread.id,
1023 occurredAt: now,
1024 payload: { ...tool, status: "completed", completedAt: now },
1025 },
1026 ],
1027 });
1028 const live = yield* pull;
1029 const otherLive = yield* other;
1030 assert.deepEqual(
1031 live.map((stored) => stored.sequence),
1032 [completed!.sequence],
1033 );
1034 // Both subscribers get one shared projection, so the live budget sizes it once.
1035 assert.strictEqual(otherLive[0], live[0]);
1036 assert.lengthOf(
1037 stringify.mock.calls.filter(([value]) => value === live[0] || value === otherLive[0]),
1038 1,
1039 );
1040 assert.isBelow(Buffer.byteLength(yield* encodeJson(live)), 4_000);
1041 for (const stored of [...replay, ...live]) {
1042 assert.equal(stored.event.type, "turn-item.updated");
1043 if (stored.event.type !== "turn-item.updated") return;
1044 assert.equal(stored.event.payload.type, "dynamic_tool");
1045 if (stored.event.payload.type !== "dynamic_tool") return;
1046 assert.notProperty(stored.event.payload, "output");
1047 }
1048 const persisted = yield* store.read({ threadId: thread.id }).pipe(Stream.runCollect);
1049 const fullEvent = persisted.find((stored) => stored.sequence === completed!.sequence)!.event;
1050 assert.equal(fullEvent.type, "turn-item.updated");
1051 if (fullEvent.type !== "turn-item.updated") return;
1052 assert.equal(fullEvent.payload.type, "dynamic_tool");
1053 if (fullEvent.payload.type !== "dynamic_tool") return;
1054 assert.deepEqual(fullEvent.payload.output, output);
1055 const detail = (yield* projections.getThreadProjection(thread.id)).turnItems.find(
1056 (item) => item.id === tool.id,
1057 )!;
1058 assert.equal(detail.type, "dynamic_tool");
1059 if (detail.type === "dynamic_tool") assert.deepEqual(detail.output, output);
1060 }),
1061 );
1062
1063 it.effect.each(["high-water", "replay"] as const)(
1064 "bounds live events while the V2 %s query is blocked",
1065 (phase) =>
1066 Effect.gen(function* () {
1067 const sink = yield* EventSink.EventSinkV2;
1068 const now = yield* DateTime.now;
1069 const thread = makeThread(ThreadId.make(`thread:blocked-${phase}`), now);
1070 const [created] = yield* sink.write({
1071 events: [threadCreatedEvent({ id: `event:blocked-${phase}:created`, thread, now })],
1072 });
1073 const readStarted = yield* Deferred.make<void>();
1074 const readClosed = yield* Deferred.make<void>();
1075 const blockRead: Statement.Transformer = (statement) => {
1076 const [query] = statement.compile();
1077 if (
1078 query.includes("FROM orchestration_events") &&
1079 query.includes("MAX(sequence)") === (phase === "high-water")
1080 ) {
1081 return Deferred.succeed(readStarted, undefined).pipe(
1082 Effect.andThen(Effect.never),
1083 Effect.ensuring(Deferred.succeed(readClosed, undefined)),
1084 Effect.as(statement),
1085 );
1086 }
1087 return Effect.succeed(statement);
1088 };
1089 const reader = yield* sink
1090 .stream({ threadId: thread.id, afterSequence: created!.sequence, bounded: true })
1091 .pipe(
1092 Stream.provideService(Statement.CurrentTransformer, blockRead),
1093 Stream.runDrain,
1094 Effect.result,
1095 Effect.forkScoped,
1096 );
1097 yield* Deferred.await(readStarted);
1098 yield* sink.write({
1099 events: Array.from({ length: LIVE_STREAM_MAX_ITEMS + 1 }, (_, index) => ({
1100 id: EventId.make(`event:blocked-${phase}:${index}`),
1101 type: "thread.metadata-updated" as const,
1102 threadId: thread.id,
1103 occurredAt: now,
1104 payload: { ...thread, title: `Updated ${index}` },
1105 })),
1106 });
1107 yield* Deferred.await(readClosed);
1108 const result = yield* Fiber.join(reader);
1109 assert.equal(result._tag, "Failure");
1110 if (result._tag === "Failure") {
1111 assert.equal(result.failure._tag, "EventSinkStreamError");
1112 assert.isTrue(isLiveStreamBufferError(result.failure.cause));
1113 }
1114 }),
1115 );
1116
1117 it.effect(
1118 "keeps internal streams subscribed while replay is blocked beyond the RPC buffer cap",
1119 () =>
1120 Effect.gen(function* () {
1121 const sink = yield* EventSink.EventSinkV2;
1122 const now = yield* DateTime.now;
1123 const thread = makeThread(ThreadId.make("thread:internal-stream-burst"), now);
1124 const [created] = yield* sink.write({
1125 events: [threadCreatedEvent({ id: "event:internal-stream-burst:created", thread, now })],
1126 });
1127 const readStarted = yield* Deferred.make<void>();
1128 const releaseRead = yield* Deferred.make<void>();
1129 const blockHighWater: Statement.Transformer = (statement) => {
1130 const [query] = statement.compile();
1131 return query.includes("MAX(sequence)") && query.includes("FROM orchestration_events")
1132 ? Deferred.succeed(readStarted, undefined).pipe(
1133 Effect.andThen(Deferred.await(releaseRead)),
1134 Effect.as(statement),
1135 )
1136 : Effect.succeed(statement);
1137 };
1138 const eventCount = LIVE_STREAM_MAX_ITEMS + 1;
1139 const reader = yield* sink
1140 .stream({ threadId: thread.id, afterSequence: created!.sequence })
1141 .pipe(
1142 Stream.provideService(Statement.CurrentTransformer, blockHighWater),
1143 Stream.take(eventCount),
1144 Stream.runCollect,
1145 Effect.forkScoped,
1146 );
1147 yield* Deferred.await(readStarted);
1148 const written = yield* sink.write({
1149 events: Array.from({ length: eventCount }, (_, index) => ({
1150 id: EventId.make(`event:internal-stream-burst:${index}`),
1151 type: "thread.metadata-updated" as const,
1152 threadId: thread.id,
1153 occurredAt: now,
1154 payload: { ...thread, title: `Updated ${index}` },
1155 })),
1156 });
1157 yield* Deferred.succeed(releaseRead, undefined);
1158 assert.deepEqual(
1159 (yield* Fiber.join(reader)).map((event) => event.sequence),
1160 written.map((event) => event.sequence),
1161 );
1162 }),
1163 );
1164
1165 it.effect("filters worker replay and live queues without losing matching events", () =>
1166 Effect.gen(function* () {
1167 const sink = yield* EventSink.EventSinkV2;
1168 const sql = yield* SqlClient.SqlClient;
1169 const now = yield* DateTime.now;
1170 const thread = makeThread(ThreadId.make("thread:filtered-worker"), now);
1171 const other = makeThread(ThreadId.make("thread:filtered-worker-other"), now);
1172 const run: OrchestrationV2Run = {
1173 id: RunId.make("run:filtered-worker"),
1174 threadId: thread.id,
1175 ordinal: 1,
1176 providerInstanceId,
1177 modelSelection,
1178 providerThreadId: null,
1179 userMessageId: MessageId.make("message:filtered-worker"),
1180 rootNodeId: null,
1181 activeAttemptId: null,
1182 status: "completed",
1183 queuePosition: null,
1184 requestedAt: now,
1185 startedAt: now,
1186 completedAt: now,
1187 checkpointId: null,
1188 contextHandoffId: null,
1189 };
1190 const runEvent = (id: string, payload: OrchestrationV2Run): OrchestrationV2DomainEvent => ({
1191 id: EventId.make(id),
1192 type: "run.updated",
1193 threadId: payload.threadId,
1194 runId: payload.id,
1195 occurredAt: now,
1196 payload,
1197 });
1198 const history = yield* sink.write({
1199 events: [
1200 threadCreatedEvent({ id: "event:filtered-worker:created", thread, now }),
1201 threadCreatedEvent({ id: "event:filtered-worker:other", thread: other, now }),
1202 runEvent("event:filtered-worker:history", run),
1203 runEvent("event:filtered-worker:other-history", {
1204 ...run,
1205 id: RunId.make("run:filtered-worker-other"),
1206 threadId: other.id,
1207 }),
1208 ],
1209 });
1210 const original = yield* sql<{ readonly payload_json: string }>`
1211 SELECT payload_json FROM orchestration_events
1212 WHERE sequence = ${history[0]!.sequence}
1213 `;
1214 yield* Effect.acquireRelease(
1215 sql`
1216 UPDATE orchestration_events SET payload_json = 'unreadable unrelated payload'
1217 WHERE sequence = ${history[0]!.sequence}
1218 `,
1219 () =>
1220 sql`
1221 UPDATE orchestration_events SET payload_json = ${original[0]!.payload_json}
1222 WHERE sequence = ${history[0]!.sequence}
1223 `.pipe(Effect.orDie),
1224 );
1225 const pull = yield* Stream.toPull(
1226 sink.stream({ threadId: thread.id, eventType: "run.updated" }),
1227 );
1228 assert.deepEqual(
1229 (yield* pull).map((stored) => stored.sequence),
1230 [history[2]!.sequence],
1231 );
1232 yield* sink.write({
1233 events: Array.from({ length: LIVE_STREAM_MAX_ITEMS + 1 }, (_, index) => ({
1234 id: EventId.make(`event:filtered-worker:output:${index}`),
1235 type: "thread.metadata-updated" as const,
1236 threadId: thread.id,
1237 occurredAt: now,
1238 payload: { ...thread, title: `Output ${index}` },
1239 })),
1240 });
1241 const live = yield* sink.write({
1242 events: [
1243 runEvent("event:filtered-worker:live:1", { ...run, status: "interrupted" }),
1244 runEvent("event:filtered-worker:live:2", { ...run, status: "failed" }),
1245 ],
1246 });
1247 assert.deepEqual(
1248 (yield* pull).map((stored) => stored.sequence),
1249 live.map((stored) => stored.sequence),
1250 );
1251 }),
1252 );
1253
1254 it.effect("paginates catch-up beyond the event-store read limit", () =>
1255 Effect.gen(function* () {
1256 const eventSink = yield* EventSink.EventSinkV2;
1257 const now = yield* DateTime.now;
1258 const threadId = ThreadId.make("thread:foundation-large-catch-up");
1259 const thread = makeThread(threadId, now);
1260 const eventCount = 1_005;
1261 const events: Array<OrchestrationV2DomainEvent> = [
1262 threadCreatedEvent({ id: "event:foundation-catch-up:0", thread, now }),
1263 ...Array.from({ length: eventCount - 1 }, (_, index) => ({
1264 id: EventId.make(`event:foundation-catch-up:${index + 1}`),
1265 type: "thread.metadata-updated" as const,
1266 threadId,
1267 providerInstanceId,
1268 occurredAt: now,
1269 payload: {
1270 ...thread,
1271 title: `Catch-up update ${index + 1}`,
1272 },
1273 })),
1274 ];
1275
1276 yield* eventSink.write({ events });
1277 const replayed = yield* eventSink.stream({ afterSequence: 0 }).pipe(
1278 Stream.take(eventCount),
1279 Stream.runCollect,
1280 Effect.map((events) => Array.from(events)),
1281 );
1282
1283 assert.lengthOf(replayed, eventCount);
1284 assert.deepEqual(
1285 replayed.map((stored) => stored.sequence),
1286 Array.from({ length: eventCount }, (_, index) => index + 1),
1287 );
1288 }),
1289 );
1290
1291 it.effect("does not lose or duplicate events while transitioning from catch-up to live", () =>
1292 Effect.gen(function* () {
1293 const eventSink = yield* EventSink.EventSinkV2;
1294 const now = yield* DateTime.now;
1295 const threadId = ThreadId.make("thread:foundation-stream-race");
1296 const thread = makeThread(threadId, now);
1297 const created = yield* eventSink.write({
1298 events: [threadCreatedEvent({ id: "event:foundation-stream-race:0", thread, now })],
1299 });
1300
1301 let afterSequence = created[0]!.sequence;
1302 // oxlint-disable-next-line t3code/no-polling-loop-in-tests -- Fixed-count writes exercise the catch-up/live handoff; each iteration awaits its event rather than polling.
1303 for (let index = 1; index <= 32; index += 1) {
1304 const nextEvent = {
1305 id: EventId.make(`event:foundation-stream-race:${index}`),
1306 type: "thread.metadata-updated" as const,
1307 threadId,
1308 providerInstanceId,
1309 occurredAt: now,
1310 payload: { ...thread, title: `Race update ${index}` },
1311 } satisfies OrchestrationV2DomainEvent;
1312 const reader = yield* eventSink
1313 .stream({ threadId, afterSequence })
1314 .pipe(Stream.runHead, Effect.forkChild);
1315 yield* Effect.yieldNow;
1316 const written = yield* eventSink.write({ events: [nextEvent] });
1317 const received = yield* Fiber.join(reader);
1318 if (Option.isNone(received)) {
1319 return yield* Effect.die("The event stream ended before delivering the live event.");
1320 }
1321 assert.equal(received.value.sequence, written[0]?.sequence);
1322 assert.equal(received.value.event.id, nextEvent.id);
1323 afterSequence = received.value.sequence;
1324 }
1325 }),
1326 );
1327
1328 it.effect("replays shared provider-session payloads across every bound thread", () =>
1329 Effect.gen(function* () {
1330 const eventSink = yield* EventSink.EventSinkV2;
1331 const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
1332 const maintenance = yield* ProjectionReplay.ProjectionReplay;
1333 const now = yield* DateTime.now;
1334 const firstThreadId = ThreadId.make("thread:foundation-shared-session:first");
1335 const secondThreadId = ThreadId.make("thread:foundation-shared-session:second");
1336 const providerSessionId = ProviderSessionId.make("provider-session:foundation:shared");
1337 const firstSession = {
1338 id: providerSessionId,
1339 driver: providerDriver,
1340 providerInstanceId,
1341 status: "ready" as const,
1342 cwd: "/workspace/first",
1343 model: modelSelection.model,
1344 capabilities: CodexProviderCapabilitiesV2,
1345 createdAt: now,
1346 updatedAt: now,
1347 lastError: null,
1348 };
1349 const secondSession = { ...firstSession, cwd: "/workspace/second" };
1350
1351 yield* eventSink.write({
1352 events: [
1353 threadCreatedEvent({
1354 id: "event:foundation-shared-session:first-thread",
1355 thread: makeThread(firstThreadId, now),
1356 now,
1357 }),
1358 {
1359 id: EventId.make("event:foundation-shared-session:first-attachment"),
1360 type: "provider-session.attached",
1361 threadId: firstThreadId,
1362 driver: providerDriver,
1363 providerInstanceId,
1364 occurredAt: now,
1365 payload: firstSession,
1366 },
1367 threadCreatedEvent({
1368 id: "event:foundation-shared-session:second-thread",
1369 thread: makeThread(secondThreadId, now),
1370 now,
1371 }),
1372 {
1373 id: EventId.make("event:foundation-shared-session:second-attachment"),
1374 type: "provider-session.attached",
1375 threadId: secondThreadId,
1376 driver: providerDriver,
1377 providerInstanceId,
1378 occurredAt: now,
1379 payload: secondSession,
1380 },
1381 ],
1382 });
1383
1384 assert.equal(
1385 (yield* projectionStore.getThreadProjection(firstThreadId)).providerSessions[0]?.cwd,
1386 secondSession.cwd,
1387 );
1388 assert.isTrue((yield* maintenance.verify).valid);
1389 assert.isTrue((yield* maintenance.rebuild).valid);
1390 }),
1391 );
1392
1393 it.effect("prunes covered history, imported v1 events, and legacy receipts", () =>
1394 Effect.gen(function* () {
1395 const eventSink = yield* EventSink.EventSinkV2;
1396 const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2;
1397 const sql = yield* SqlClient.SqlClient;
1398 const projections = yield* ProjectionStore.ProjectionStoreV2;
1399 const now = yield* DateTime.now;
1400 const nowIso = DateTime.formatIso(now);
1401 const threadId = ThreadId.make("thread:foundation-compact");
1402 const thread = makeThread(threadId, now);
1403 const messageId = MessageId.make("message:foundation-compact");
1404 const threadStateEvent = (
1405 suffix: string,
1406 type: "thread.visited" | "thread.metadata-updated" | "thread.tags-updated",
1407 ): OrchestrationV2DomainEvent => ({
1408 id: EventId.make(`event:foundation-compact:${suffix}`),
1409 type,
1410 threadId,
1411 providerInstanceId,
1412 occurredAt: now,
1413 payload: { ...thread, lastVisitedAt: now },
1414 });
1415 const messageEvent = (suffix: string, text: string): OrchestrationV2DomainEvent => ({
1416 id: EventId.make(`event:foundation-compact:${suffix}`),
1417 type: "message.updated",
1418 threadId,
1419 occurredAt: now,
1420 payload: {
1421 createdBy: "user",
1422 creationSource: "web",
1423 id: messageId,
1424 threadId,
1425 runId: null,
1426 nodeId: null,
1427 role: "user",
1428 text,
1429 attachments: [],
1430 streaming: false,
1431 createdAt: now,
1432 updatedAt: now,
1433 },
1434 });
1435 const nodeId = NodeId.make("node:foundation-compact");
1436 const nodeEvent = (
1437 suffix: string,
1438 status: "running" | "completed",
1439 ): OrchestrationV2DomainEvent => ({
1440 id: EventId.make(`event:foundation-compact:${suffix}`),
1441 type: "node.updated",
1442 threadId,
1443 occurredAt: now,
1444 payload: {
1445 id: nodeId,
1446 threadId,
1447 runId: null,
1448 parentNodeId: null,
1449 rootNodeId: nodeId,
1450 kind: "assistant_message",
1451 status,
1452 countsForRun: false,
1453 providerThreadId: null,
1454 providerTurnId: null,
1455 nativeItemRef: null,
1456 runtimeRequestId: null,
1457 checkpointScopeId: null,
1458 startedAt: now,
1459 completedAt: status === "completed" ? now : null,
1460 },
1461 });
1462 const itemEvent = (suffix: string, text: string): OrchestrationV2DomainEvent => ({
1463 id: EventId.make(`event:foundation-compact:${suffix}`),
1464 type: "turn-item.updated",
1465 threadId,
1466 occurredAt: now,
1467 payload: {
1468 id: TurnItemId.make("turn-item:foundation-compact"),
1469 threadId,
1470 runId: null,
1471 nodeId: null,
1472 providerThreadId: null,
1473 providerTurnId: null,
1474 nativeItemRef: null,
1475 parentItemId: null,
1476 ordinal: 1,
1477 status: "completed",
1478 title: null,
1479 startedAt: now,
1480 completedAt: now,
1481 updatedAt: now,
1482 type: "assistant_message",
1483 messageId,
1484 text,
1485 streaming: false,
1486 },
1487 });
1488
1489 yield* eventSink.write({
1490 events: [
1491 threadCreatedEvent({ id: "event:foundation-compact:create", thread, now }),
1492 threadStateEvent("meta", "thread.metadata-updated"),
1493 threadStateEvent("visit-1", "thread.visited"),
1494 messageEvent("message-1", "streaming"),
1495 nodeEvent("node-1", "running"),
1496 itemEvent("item-1", "streaming"),
1497 itemEvent("item-middle", "still streaming"),
1498 ...Array.from({ length: 501 }, (_, index) =>
1499 threadStateEvent(`history-${index}`, "thread.visited"),
1500 ),
1501 threadStateEvent("tags-1", "thread.tags-updated"),
1502 threadStateEvent("visit-2", "thread.visited"),
1503 messageEvent("message-2", "final"),
1504 nodeEvent("node-2", "completed"),
1505 itemEvent("item-2", "final"),
1506 // Older clients skip tag events, so these must not remove the rename above.
1507 threadStateEvent("tags-2", "thread.tags-updated"),
1508 ],
1509 });
1510 const beforeCompaction = yield* projections.getThreadProjection(threadId);
1511
1512 const importedV1ThreadId = "thread:foundation-compact-v1-imported";
1513 const pendingV1ThreadId = "thread:foundation-compact-v1-pending";
1514 const insertV1Event = (threadIdValue: string, version: number) => sql`
1515 INSERT INTO orchestration_events (
1516 event_id, aggregate_kind, stream_id, stream_version, event_type,
1517 occurred_at, actor_kind, payload_json, metadata_json, application_event_version
1518 )
1519 VALUES (
1520 ${`event:v1:${threadIdValue}:${version}`}, 'thread', ${threadIdValue}, ${version},
1521 'thread.message-appended', ${nowIso}, 'user', '{}', '{}', 1
1522 )
1523 `;
1524 yield* insertV1Event(importedV1ThreadId, 1);
1525 yield* insertV1Event(importedV1ThreadId, 2);
1526 yield* insertV1Event(pendingV1ThreadId, 1);
1527 yield* sql`
1528 INSERT INTO orchestration_v2_legacy_imports (
1529 thread_id, source_updated_at, shell_imported_at, transcript_imported_at,
1530 imported_message_count, last_error
1531 )
1532 VALUES
1533 (${importedV1ThreadId}, ${nowIso}, ${nowIso}, ${nowIso}, 2, NULL),
1534 (${pendingV1ThreadId}, ${nowIso}, ${nowIso}, NULL, 0, NULL)
1535 ON CONFLICT(thread_id) DO NOTHING
1536 `;
1537 yield* sql`
1538 INSERT INTO orchestration_command_receipts (
1539 command_id, aggregate_kind, aggregate_id, accepted_at, result_sequence, status, command_type
1540 )
1541 VALUES
1542 ('command:foundation-compact:legacy', 'thread', ${importedV1ThreadId}, ${nowIso}, 1, 'accepted', 'legacy'),
1543 ('command:foundation-compact:pending', 'thread', ${pendingV1ThreadId}, ${nowIso}, 1, 'accepted', 'legacy')
1544 ON CONFLICT(command_id) DO NOTHING
1545 `;
1546
1547 // The first pass only marks the head and drops imported legacy rows.
1548 const marked = yield* maintenance.pruneEventStore();
1549 yield* TestClock.adjust("1 day");
1550 const summary = yield* maintenance.pruneEventStore();
1551 assert.isAtLeast(summary.deletedEventCount, 504);
1552 assert.equal(marked.deletedReceiptCount + summary.deletedReceiptCount, 1);
1553
1554 const remaining = yield* sql<{ readonly event_id: string }>`
1555 SELECT event_id
1556 FROM orchestration_events
1557 WHERE aggregate_kind = 'thread'
1558 AND stream_id = ${threadId}
1559 ORDER BY sequence ASC
1560 `;
1561 assert.deepEqual(
1562 remaining.map((row) => row.event_id),
1563 ["event:foundation-compact:create", "event:foundation-compact:tags-2"],
1564 );
1565
1566 const remainingV1 = yield* sql<{ readonly stream_id: string }>`
1567 SELECT stream_id
1568 FROM orchestration_events
1569 WHERE application_event_version = 1
1570 AND stream_id IN (${importedV1ThreadId}, ${pendingV1ThreadId})
1571 `;
1572 assert.deepEqual(
1573 remainingV1.map((row) => row.stream_id),
1574 [pendingV1ThreadId],
1575 );
1576
1577 const remainingReceipts = yield* sql<{ readonly command_id: string }>`
1578 SELECT command_id
1579 FROM orchestration_command_receipts
1580 WHERE command_id IN ('command:foundation-compact:legacy', 'command:foundation-compact:pending')
1581 `;
1582 assert.deepEqual(
1583 remainingReceipts.map((row) => row.command_id),
1584 ["command:foundation-compact:pending"],
1585 );
1586
1587 assert.deepEqual(yield* projections.getThreadProjection(threadId), beforeCompaction);
1588 }),
1589 );
1590
1591 it.effect("verifies and rebuilds projections with cross-thread subagent relations", () =>
1592 Effect.gen(function* () {
1593 const eventSink = yield* EventSink.EventSinkV2;
1594 const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
1595 const maintenance = yield* ProjectionReplay.ProjectionReplay;
1596 const sql = yield* SqlClient.SqlClient;
1597 const now = yield* DateTime.now;
1598 const parentThreadId = ThreadId.make("thread:foundation-cross-thread:parent");
1599 const childThreadId = ThreadId.make("thread:foundation-cross-thread:child");
1600 const childProviderThreadId = ProviderThreadId.make(
1601 "provider-thread:foundation-cross-thread:child",
1602 );
1603 const subagentId = NodeId.make("subagent:foundation-cross-thread");
1604 const spawnTransferId = ContextTransferId.make("transfer:foundation-cross-thread:spawn");
1605 const resultTransferId = ContextTransferId.make("transfer:foundation-cross-thread:result");
1606 const parentThread = makeThread(parentThreadId, now);
1607 const childThread = {
1608 ...makeThread(childThreadId, now),
1609 createdBy: "agent" as const,
1610 creationSource: "provider" as const,
1611 lineage: {
1612 parentThreadId,
1613 relationshipToParent: "subagent" as const,
1614 rootThreadId: parentThreadId,
1615 },
1616 };
1617
1618 yield* eventSink.write({
1619 events: [
1620 threadCreatedEvent({
1621 id: "event:foundation-cross-thread:parent",
1622 thread: parentThread,
1623 now,
1624 }),
1625 threadCreatedEvent({
1626 id: "event:foundation-cross-thread:child",
1627 thread: childThread,
1628 now,
1629 }),
1630 {
1631 id: EventId.make("event:foundation-cross-thread:subagent"),
1632 type: "subagent.updated",
1633 threadId: parentThreadId,
1634 nodeId: subagentId,
1635 providerInstanceId,
1636 occurredAt: now,
1637 payload: {
1638 id: subagentId,
1639 threadId: parentThreadId,
1640 runId: null,
1641 parentNodeId: NodeId.make("node:foundation-cross-thread:parent"),
1642 origin: "app_owned",
1643 createdBy: "agent",
1644 driver: providerDriver,
1645 providerInstanceId,
1646 providerThreadId: childProviderThreadId,
1647 childThreadId,
1648 nativeTaskRef: null,
1649 prompt: "Inspect the child flow",
1650 title: "Cross-thread child",
1651 model: modelSelection.model,
1652 status: "completed",
1653 result: "done",
1654 startedAt: now,
1655 completedAt: now,
1656 updatedAt: now,
1657 },
1658 },
1659 {
1660 id: EventId.make("event:foundation-cross-thread:spawn-transfer"),
1661 type: "context-transfer.created",
1662 threadId: childThreadId,
1663 providerInstanceId,
1664 occurredAt: now,
1665 payload: {
1666 id: spawnTransferId,
1667 type: "subagent_spawn",
1668 sourceThreadId: parentThreadId,
1669 targetThreadId: childThreadId,
1670 sourcePoint: { threadId: parentThreadId },
1671 basePoint: null,
1672 sourceProviderInstanceId: providerInstanceId,
1673 targetProviderInstanceId: providerInstanceId,
1674 targetRunId: null,
1675 status: "consumed",
1676 resolution: null,
1677 createdBy: "agent",
1678 error: null,
1679 createdAt: now,
1680 updatedAt: now,
1681 consumedAt: now,
1682 },
1683 },
1684 {
1685 id: EventId.make("event:foundation-cross-thread:provider-thread"),
1686 type: "provider-thread.updated",
1687 threadId: parentThreadId,
1688 driver: providerDriver,
1689 providerInstanceId,
1690 occurredAt: now,
1691 payload: {
1692 id: childProviderThreadId,
1693 driver: providerDriver,
1694 providerInstanceId,
1695 providerSessionId: null,
1696 appThreadId: childThreadId,
1697 ownerNodeId: null,
1698 nativeThreadRef: null,
1699 nativeConversationHeadRef: null,
1700 status: "idle",
1701 firstRunOrdinal: 1,
1702 lastRunOrdinal: 1,
1703 handoffIds: [],
1704 forkedFrom: null,
1705 createdAt: now,
1706 updatedAt: now,
1707 },
1708 },
1709 {
1710 id: EventId.make("event:foundation-cross-thread:result-transfer"),
1711 type: "context-transfer.created",
1712 threadId: parentThreadId,
1713 providerInstanceId,
1714 occurredAt: now,
1715 payload: {
1716 id: resultTransferId,
1717 type: "subagent_result",
1718 sourceThreadId: childThreadId,
1719 targetThreadId: parentThreadId,
1720 sourcePoint: { threadId: childThreadId },
1721 basePoint: null,
1722 sourceProviderInstanceId: providerInstanceId,
1723 targetProviderInstanceId: providerInstanceId,
1724 targetRunId: null,
1725 status: "consumed",
1726 resolution: null,
1727 createdBy: "system",
1728 error: null,
1729 createdAt: now,
1730 updatedAt: now,
1731 consumedAt: now,
1732 },
1733 },
1734 ],
1735 });
1736
1737 const assertCrossThreadProjection = Effect.gen(function* () {
1738 const parent = yield* projectionStore.getThreadProjection(parentThreadId);
1739 const child = yield* projectionStore.getThreadProjection(childThreadId);
1740 const expectedTransferIds = [spawnTransferId, resultTransferId].toSorted();
1741 assert.deepEqual(
1742 parent.contextTransfers.map((transfer) => transfer.id).toSorted(),
1743 expectedTransferIds,
1744 );
1745 assert.deepEqual(
1746 child.contextTransfers.map((transfer) => transfer.id).toSorted(),
1747 expectedTransferIds,
1748 );
1749 assert.deepEqual(
1750 parent.providerThreads.map((providerThread) => providerThread.id),
1751 [childProviderThreadId],
1752 );
1753 assert.equal(child.thread.activeProviderThreadId, childProviderThreadId);
1754 });
1755
1756 yield* assertCrossThreadProjection;
1757 assert.isTrue((yield* maintenance.verify).valid);
1758 yield* sql`
1759 UPDATE orchestration_v2_projection_provider_threads SET payload_json = '{}'
1760 WHERE provider_thread_id = ${childProviderThreadId}
1761 `;
1762 assert.deepEqual(
1763 new Set((yield* maintenance.verify).unreadableThreadIds),
1764 new Set([parentThreadId, childThreadId]),
1765 );
1766 assert.isTrue((yield* maintenance.rebuild).valid);
1767 yield* assertCrossThreadProjection;
1768 }),
1769 );
1770
1771 it.effect(
1772 "rolls back events, projections, receipts, and effects after a projection failure",
1773 () =>
1774 Effect.gen(function* () {
1775 const eventSink = yield* EventSink.EventSinkV2;
1776 const eventStore = yield* EventStore.EventStoreV2;
1777 const receipts = yield* CommandReceiptStore.CommandReceiptStoreV2;
1778 const outbox = yield* EffectOutbox.EffectOutboxV2;
1779 const sql = yield* SqlClient.SqlClient;
1780 const now = yield* DateTime.now;
1781 const commandId = CommandId.make("command:foundation-atomic-failure");
1782 const threadId = ThreadId.make("thread:foundation-atomic-failure");
1783 const scopeId = CheckpointScopeId.make("scope:foundation-atomic-failure");
1784 const checkpoint = (index: number) => ({
1785 id: CheckpointId.make(`checkpoint:foundation-atomic-failure:${index}`),
1786 threadId,
1787 scopeId,
1788 runId: null,
1789 nodeId: NodeId.make("node:foundation-atomic-failure"),
1790 parentCheckpointId: null,
1791 ordinalWithinScope: 1,
1792 appRunOrdinal: null,
1793 ref: CheckpointRef.make(`checkpoint-ref:foundation-atomic-failure:${index}`),
1794 status: "ready" as const,
1795 files: [],
1796 capturedAt: now,
1797 });
1798 const events = [1, 2].map(
1799 (index) =>
1800 ({
1801 id: EventId.make(`event:foundation-atomic-failure:${index}`),
1802 type: "checkpoint.captured",
1803 threadId,
1804 occurredAt: now,
1805 payload: checkpoint(index),
1806 }) satisfies OrchestrationV2DomainEvent,
1807 );
1808
1809 const exit = yield* Effect.exit(
1810 eventSink.commitCommand({
1811 commandId,
1812 threadId,
1813 commandType: "checkpoint.atomicity-test",
1814 acceptedAt: now,
1815 events,
1816 effects: [
1817 {
1818 id: "effect:foundation-atomic-failure",
1819 commandId,
1820 threadId,
1821 request: {
1822 type: "provider-turn.start",
1823 runId: RunId.make("run:foundation-atomic-failure"),
1824 },
1825 },
1826 ],
1827 }),
1828 );
1829 assert.equal(exit._tag, "Failure");
1830 assert.isTrue(Option.isNone(yield* receipts.getByCommandId(commandId)));
1831 assert.deepEqual(yield* outbox.listByCommandId(commandId), []);
1832 assert.deepEqual(
1833 yield* eventStore.readByCommandId({ commandId }).pipe(
1834 Stream.runCollect,
1835 Effect.map((events) => Array.from(events)),
1836 ),
1837 [],
1838 );
1839 const checkpointRows = yield* sql<{ readonly count: number }>`
1840 SELECT COUNT(*) AS count
1841 FROM orchestration_v2_projection_checkpoints
1842 WHERE thread_id = ${threadId}
1843 `;
1844 assert.equal(checkpointRows[0]?.count, 0);
1845 }),
1846 );
1847
1848 it.effect("replays every command event across bounded persistence pages", () =>
1849 Effect.gen(function* () {
1850 const sql = yield* SqlClient.SqlClient;
1851 const eventStore = yield* EventStore.EventStoreV2;
1852 const eventSink = yield* EventSink.EventSinkV2;
1853 const now = yield* DateTime.now;
1854 const occurredAt = DateTime.formatIso(now);
1855 const commandId = CommandId.make("command:paged-command-replay");
1856 const exactPageCommandId = CommandId.make("command:paged-command-replay:exact");
1857 const emptyCommandId = CommandId.make("command:paged-command-replay:empty");
1858 const threadId = "thread:paged-command-replay";
1859
1860 const eventRow = (
1861 ordinal: number,
1862 input: {
1863 readonly commandId?: string | null;
1864 readonly aggregateKind?: "project" | "thread";
1865 readonly streamId?: string;
1866 readonly version?: number;
1867 } = {},
1868 ) => ({
1869 event_id: `event:paged-command-replay:${ordinal}`,
1870 aggregate_kind: input.aggregateKind ?? "thread",
1871 stream_id: input.streamId ?? threadId,
1872 stream_version: ordinal,
1873 event_type: "provider-session.detached",
1874 occurred_at: occurredAt,
1875 command_id: input.commandId ?? null,
1876 causation_event_id: null,
1877 correlation_id: null,
1878 actor_kind: "server",
1879 payload_json: JSON.stringify({
1880 providerSessionId: `session:paged-command-replay:${ordinal}`,
1881 detachedAt: occurredAt,
1882 }),
1883 metadata_json: "{}",
1884 application_event_version: input.version ?? 2,
1885 });
1886
1887 const matchingCount = 1_001;
1888 const rows: Array<ReturnType<typeof eventRow>> = [];
1889 let ordinal = 0;
1890 for (let index = 0; index < matchingCount; index += 1) {
1891 rows.push(eventRow(++ordinal, { commandId }));
1892 if (index % 2 === 0) {
1893 rows.push(eventRow(++ordinal, { commandId: "command:unrelated" }));
1894 }
1895 if (index % 5 === 0) {
1896 rows.push(eventRow(++ordinal));
1897 rows.push(eventRow(++ordinal, { commandId, version: 1 }));
1898 rows.push(
1899 eventRow(++ordinal, {
1900 commandId,
1901 aggregateKind: "project",
1902 streamId: `project:paged-command-replay:${index}`,
1903 }),
1904 );
1905 }
1906 }
1907 for (let index = 0; index < 500; index += 1) {
1908 rows.push(eventRow(++ordinal, { commandId: exactPageCommandId }));
1909 }
1910 const inserted = yield* Effect.forEach(
1911 Arr.chunksOf(rows, 400),
1912 (chunk) =>
1913 sql<{
1914 readonly sequence: number;
1915 readonly command_id: string | null;
1916 readonly aggregate_kind: string;
1917 readonly application_event_version: number;
1918 }>`
1919 INSERT INTO orchestration_events ${sql.insert(chunk)}
1920 RETURNING sequence, command_id, aggregate_kind, application_event_version
1921 `,
1922 { concurrency: 1 },
1923 ).pipe(Effect.map((chunks) => chunks.flat()));
1924 const expectedSequences = inserted
1925 .filter(
1926 (row) =>
1927 row.command_id === commandId &&
1928 row.aggregate_kind === "thread" &&
1929 row.application_event_version === 2,
1930 )
1931 .map((row) => row.sequence);
1932 assert.lengthOf(expectedSequences, matchingCount);
1933
1934 yield* sql`
1935 INSERT INTO orchestration_command_receipts (
1936 command_id, aggregate_kind, aggregate_id, accepted_at, result_sequence, status, command_type
1937 )
1938 VALUES (
1939 ${commandId}, 'thread', ${threadId}, ${occurredAt},
1940 ${expectedSequences.at(-1)!}, 'accepted', 'thread.create'
1941 )
1942 `;
1943
1944 const pageLimits: Array<number> = [];
1945 const recordReads: Statement.Transformer = (statement) => {
1946 const [query, params] = statement.compile();
1947 if (query.includes("FROM orchestration_events") && query.includes("command_id")) {
1948 const limit = params.at(-1);
1949 if (typeof limit === "number") {
1950 pageLimits.push(limit);
1951 }
1952 }
1953 return Effect.succeed(statement);
1954 };
1955 const collectByCommandId = (id: CommandId) =>
1956 eventStore.readByCommandId({ commandId: id }).pipe(
1957 Stream.provideService(Statement.CurrentTransformer, recordReads),
1958 Stream.runCollect,
1959 Effect.map((events) => Array.from(events)),
1960 );
1961
1962 const replayed = yield* collectByCommandId(commandId);
1963 assert.deepEqual(
1964 replayed.map((stored) => stored.sequence),
1965 expectedSequences,
1966 );
1967 assert.deepEqual(pageLimits, [500, 500, 500]);
1968
1969 pageLimits.length = 0;
1970 const exactPage = yield* collectByCommandId(exactPageCommandId);
1971 assert.lengthOf(exactPage, 500);
1972 assert.deepEqual(pageLimits, [500, 500]);
1973
1974 pageLimits.length = 0;
1975 const empty = yield* collectByCommandId(emptyCommandId);
1976 assert.lengthOf(empty, 0);
1977 assert.deepEqual(pageLimits, [500]);
1978
1979 pageLimits.length = 0;
1980 const retried = yield* eventSink
1981 .commitCommand({
1982 commandId,
1983 threadId: ThreadId.make(threadId),
1984 commandType: "thread.create",
1985 acceptedAt: now,
1986 events: [
1987 threadCreatedEvent({
1988 id: "event:paged-command-replay:retry",
1989 thread: makeThread(ThreadId.make(threadId), now),
1990 now,
1991 }),
1992 ],
1993 effects: [],
1994 })
1995 .pipe(Effect.provideService(Statement.CurrentTransformer, recordReads));
1996 assert.isFalse(retried.committed);
1997 assert.equal(retried.receipt.resultSequence, expectedSequences.at(-1));
1998 assert.deepEqual(
1999 retried.storedEvents.map((stored) => stored.sequence),
2000 expectedSequences,
2001 );
2002 assert.deepEqual(pageLimits, [500, 500, 500]);
2003 }),
2004 );
2005
2006 it.effect("keeps one durable effect across command retries and executes it after recovery", () =>
2007 Effect.gen(function* () {
2008 const eventSink = yield* EventSink.EventSinkV2;
2009 const outbox = yield* EffectOutbox.EffectOutboxV2;
2010 const now = yield* DateTime.now;
2011 const commandId = CommandId.make("command:foundation-effect-recovery");
2012 const threadId = ThreadId.make("thread:foundation-effect-recovery");
2013 const thread = makeThread(threadId, now);
2014 const event = threadCreatedEvent({
2015 id: "event:foundation-effect-recovery",
2016 thread,
2017 now,
2018 });
2019 const effect = {
2020 id: "effect:foundation-effect-recovery",
2021 commandId,
2022 threadId,
2023 request: {
2024 type: "provider-turn.start" as const,
2025 runId: RunId.make("run:foundation-effect-recovery"),
2026 },
2027 };
2028
2029 const first = yield* eventSink.commitCommand({
2030 commandId,
2031 threadId,
2032 commandType: "foundation.effect-recovery",
2033 acceptedAt: now,
2034 events: [event],
2035 effects: [effect],
2036 });
2037 const retry = yield* eventSink.commitCommand({
2038 commandId,
2039 threadId,
2040 commandType: "foundation.effect-recovery",
2041 acceptedAt: now,
2042 events: [event],
2043 effects: [effect],
2044 });
2045
2046 assert.isTrue(first.committed);
2047 assert.isFalse(retry.committed);
2048 assert.equal(retry.receipt.resultSequence, first.receipt.resultSequence);
2049 assert.lengthOf(retry.storedEvents, 1);
2050 assert.lengthOf(yield* outbox.listByCommandId(commandId), 1);
2051
2052 const executionCount = yield* Ref.make(0);
2053 const layerExecutor = Layer.succeed(
2054 EffectWorker.OrchestrationEffectExecutorV2,
2055 EffectWorker.OrchestrationEffectExecutorV2.of({
2056 execute: () => Ref.update(executionCount, (count) => count + 1),
2057 }),
2058 );
2059 const layerWorker = EffectWorker.layerWithOptions({ workerId: "recovery-worker" }).pipe(
2060 Layer.provide(
2061 Layer.merge(Layer.succeed(EffectOutbox.EffectOutboxV2, outbox), layerExecutor),
2062 ),
2063 );
2064 yield* Effect.gen(function* () {
2065 const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
2066 assert.isTrue(yield* worker.runOnce());
2067 assert.isFalse(yield* worker.runOnce());
2068 }).pipe(Effect.provide(layerWorker));
2069
2070 assert.equal(yield* Ref.get(executionCount), 1);
2071 const storedEffect = yield* outbox.get(effect.id);
2072 assert.isTrue(Option.isSome(storedEffect));
2073 if (Option.isSome(storedEffect)) {
2074 assert.equal(storedEffect.value.status, "succeeded");
2075 }
2076 }),
2077 );
2078
2079 it.effect("does not publish a stale provider start after an interrupt wins", () =>
2080 Effect.gen(function* () {
2081 const eventSink = yield* EventSink.EventSinkV2;
2082 const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
2083 const outbox = yield* EffectOutbox.EffectOutboxV2;
2084 const now = yield* DateTime.now;
2085 const threadId = ThreadId.make("thread:foundation-stale-provider-start");
2086 const runId = RunId.make("run:foundation-stale-provider-start");
2087 const attemptId = RunAttemptId.make("run-attempt:foundation-stale-provider-start");
2088 const thread = makeThread(threadId, now);
2089 const startingRun: OrchestrationV2Run = {
2090 id: runId,
2091 threadId,
2092 ordinal: 1,
2093 providerInstanceId,
2094 modelSelection,
2095 providerThreadId: null,
2096 userMessageId: MessageId.make("message:foundation-stale-provider-start"),
2097 rootNodeId: null,
2098 activeAttemptId: attemptId,
2099 status: "starting",
2100 queuePosition: null,
2101 requestedAt: now,
2102 startedAt: null,
2103 completedAt: null,
2104 checkpointId: null,
2105 contextHandoffId: null,
2106 };
2107 yield* eventSink.write({
2108 events: [
2109 threadCreatedEvent({
2110 id: "event:foundation-stale-provider-start:thread",
2111 thread,
2112 now,
2113 }),
2114 {
2115 id: EventId.make("event:foundation-stale-provider-start:run"),
2116 type: "run.created",
2117 threadId,
2118 runId,
2119 providerInstanceId,
2120 occurredAt: now,
2121 payload: startingRun,
2122 },
2123 ],
2124 });
2125
2126 const captureEffect = {
2127 id: "effect:foundation-current-capture",
2128 commandId: CommandId.make("command:foundation-current-capture"),
2129 threadId,
2130 request: {
2131 type: "checkpoint.capture" as const,
2132 runId,
2133 scopeId: CheckpointScopeId.make("scope:foundation-current-capture"),
2134 },
2135 };
2136 assert.isTrue(
2137 (yield* eventSink.writeIfRunCurrent({
2138 threadId,
2139 runId,
2140 activeAttemptId: attemptId,
2141 expectedStatus: "starting",
2142 events: [],
2143 effects: [captureEffect],
2144 })).committed,
2145 );
2146 assert.isTrue(Option.isSome(yield* outbox.get(captureEffect.id)));
2147 yield* outbox.awaitAvailable;
2148 const staleCaptureEffect = { ...captureEffect, id: "effect:foundation-stale-capture" };
2149
2150 const reachedPrecommitGap = yield* Deferred.make<void>();
2151 const releaseStaleStart = yield* Deferred.make<void>();
2152 const providerStartCount = yield* Ref.make(0);
2153 const staleStartFiber = yield* Effect.gen(function* () {
2154 yield* Deferred.succeed(reachedPrecommitGap, undefined);
2155 yield* Deferred.await(releaseStaleStart);
2156 const result = yield* eventSink.writeIfRunCurrent({
2157 threadId,
2158 runId,
2159 activeAttemptId: attemptId,
2160 expectedStatus: "starting",
2161 effects: [staleCaptureEffect],
2162 events: [
2163 {
2164 id: EventId.make("event:foundation-stale-provider-start:running"),
2165 type: "run.updated",
2166 threadId,
2167 runId,
2168 providerInstanceId,
2169 occurredAt: now,
2170 payload: { ...startingRun, status: "running", startedAt: now },
2171 },
2172 ],
2173 });
2174 if (result.committed) {
2175 yield* Ref.update(providerStartCount, (count) => count + 1);
2176 }
2177 return result;
2178 }).pipe(Effect.forkChild);
2179
2180 yield* Deferred.await(reachedPrecommitGap);
2181 const interruptedAt = yield* DateTime.now;
2182 yield* eventSink.write({
2183 events: [
2184 {
2185 id: EventId.make("event:foundation-stale-provider-start:cancelled"),
2186 type: "run.updated",
2187 threadId,
2188 runId,
2189 providerInstanceId,
2190 occurredAt: interruptedAt,
2191 payload: {
2192 ...startingRun,
2193 status: "cancelled",
2194 completedAt: interruptedAt,
2195 },
2196 },
2197 ],
2198 });
2199 yield* Deferred.succeed(releaseStaleStart, undefined);
2200
2201 const staleResult = yield* Fiber.join(staleStartFiber);
2202 assert.isFalse(staleResult.committed);
2203 assert.isTrue(Option.isNone(yield* outbox.get(staleCaptureEffect.id)));
2204 assert.deepEqual(staleResult.storedEvents, []);
2205 assert.equal(yield* Ref.get(providerStartCount), 0);
2206 const projection = yield* projectionStore.getThreadProjection(threadId);
2207 assert.equal(projection.runs[0]?.status, "cancelled");
2208 }).pipe(Effect.provide(Layer.fresh(layerTest))),
2209 );
2210
2211 it.effect("refuses a run-current write once an unpaired interrupt request is stored", () =>
2212 Effect.gen(function* () {
2213 const eventSink = yield* EventSink.EventSinkV2;
2214 const now = yield* DateTime.now;
2215 const threadId = ThreadId.make("thread:foundation-interrupt-guard");
2216 const runId = RunId.make("run:foundation-interrupt-guard");
2217 const attemptId = RunAttemptId.make("run-attempt:foundation-interrupt-guard");
2218 const requestId = TurnItemId.make("turn-item:foundation-interrupt-guard:request");
2219 const resultId = TurnItemId.make("turn-item:foundation-interrupt-guard:result");
2220 const run: OrchestrationV2Run = {
2221 id: runId,
2222 threadId,
2223 ordinal: 1,
2224 providerInstanceId,
2225 modelSelection,
2226 providerThreadId: null,
2227 userMessageId: MessageId.make("message:foundation-interrupt-guard"),
2228 rootNodeId: null,
2229 activeAttemptId: attemptId,
2230 status: "running",
2231 queuePosition: null,
2232 requestedAt: now,
2233 startedAt: now,
2234 completedAt: null,
2235 checkpointId: null,
2236 contextHandoffId: null,
2237 };
2238 yield* eventSink.write({
2239 events: [
2240 threadCreatedEvent({
2241 id: "event:foundation-interrupt-guard:thread",
2242 thread: makeThread(threadId, now),
2243 now,
2244 }),
2245 {
2246 id: EventId.make("event:foundation-interrupt-guard:run"),
2247 type: "run.created",
2248 threadId,
2249 runId,
2250 providerInstanceId,
2251 occurredAt: now,
2252 payload: run,
2253 },
2254 ],
2255 });
2256 const signalItem = (id: TurnItemId, type: "run_interrupt_request" | "run_interrupt_result") =>
2257 ({
2258 id: EventId.make(`event:${id}`),
2259 type: "turn-item.updated",
2260 threadId,
2261 runId,
2262 occurredAt: now,
2263 payload: {
2264 id,
2265 threadId,
2266 runId,
2267 nodeId: null,
2268 providerThreadId: null,
2269 providerTurnId: null,
2270 nativeItemRef: null,
2271 parentItemId: null,
2272 ordinal: 1,
2273 status: "completed",
2274 title: "Interrupt",
2275 startedAt: now,
2276 completedAt: now,
2277 updatedAt: now,
2278 type,
2279 message: "Stopped",
2280 },
2281 }) as OrchestrationV2DomainEvent;
2282 const completeRun = (suffix: string) =>
2283 eventSink.writeIfRunCurrent({
2284 threadId,
2285 runId,
2286 activeAttemptId: attemptId,
2287 expectedStatus: "running",
2288 unlessInterruptRequested: { requestId, resultId },
2289 events: [
2290 {
2291 id: EventId.make(`event:foundation-interrupt-guard:waiting:${suffix}`),
2292 type: "run.updated",
2293 threadId,
2294 runId,
2295 providerInstanceId,
2296 occurredAt: now,
2297 payload: { ...run, status: "waiting" },
2298 },
2299 ],
2300 });
2301
2302 yield* eventSink.write({ events: [signalItem(requestId, "run_interrupt_request")] });
2303 const refused = yield* completeRun("refused");
2304 assert.isFalse(refused.committed);
2305 assert.deepEqual(refused.storedEvents, []);
2306
2307 yield* eventSink.write({ events: [signalItem(resultId, "run_interrupt_result")] });
2308 assert.isTrue((yield* completeRun("paired")).committed);
2309 }).pipe(Effect.provide(Layer.fresh(layerTest))),
2310 );
2311
2312 it.effect("guards post-terminal provider-thread writes by attempt and run ordinal", () =>
2313 Effect.gen(function* () {
2314 const eventSink = yield* EventSink.EventSinkV2;
2315 const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
2316 const now = yield* DateTime.now;
2317 const threadId = ThreadId.make("thread:foundation-provider-thread-owner");
2318 const runId = RunId.make("run:foundation-provider-thread-owner");
2319 const attemptId = RunAttemptId.make("attempt:foundation-provider-thread-owner");
2320 const replacementAttemptId = RunAttemptId.make(
2321 "attempt:foundation-provider-thread-owner:replacement",
2322 );
2323 const rootNodeId = NodeId.make("node:foundation-provider-thread-owner");
2324 const providerThreadId = ProviderThreadId.make(
2325 "provider-thread:foundation-provider-thread-owner",
2326 );
2327 const thread = makeThread(threadId, now);
2328 const run: OrchestrationV2Run = {
2329 id: runId,
2330 threadId,
2331 ordinal: 1,
2332 providerInstanceId,
2333 modelSelection,
2334 providerThreadId,
2335 userMessageId: MessageId.make("message:foundation-provider-thread-owner"),
2336 rootNodeId,
2337 activeAttemptId: attemptId,
2338 status: "completed",
2339 queuePosition: null,
2340 requestedAt: now,
2341 startedAt: now,
2342 completedAt: now,
2343 checkpointId: null,
2344 contextHandoffId: null,
2345 };
2346 const baseProviderThread = {
2347 id: providerThreadId,
2348 driver: providerDriver,
2349 providerInstanceId,
2350 providerSessionId: null,
2351 appThreadId: threadId,
2352 ownerNodeId: null,
2353 nativeThreadRef: null,
2354 nativeConversationHeadRef: null,
2355 status: "idle" as const,
2356 firstRunOrdinal: 1,
2357 lastRunOrdinal: 1,
2358 handoffIds: [] as const,
2359 forkedFrom: null,
2360 pendingBackgroundTasks: [
2361 { taskId: "bg-owner", description: "sleep 20", kind: "command" as const },
2362 ],
2363 createdAt: now,
2364 updatedAt: now,
2365 };
2366
2367 yield* eventSink.write({
2368 events: [
2369 threadCreatedEvent({
2370 id: "event:foundation-provider-thread-owner:thread",
2371 thread,
2372 now,
2373 }),
2374 {
2375 id: EventId.make("event:foundation-provider-thread-owner:run"),
2376 type: "run.created",
2377 threadId,
2378 runId,
2379 nodeId: rootNodeId,
2380 providerInstanceId,
2381 occurredAt: now,
2382 payload: run,
2383 },
2384 {
2385 id: EventId.make("event:foundation-provider-thread-owner:provider-thread"),
2386 type: "provider-thread.updated",
2387 threadId,
2388 driver: providerDriver,
2389 providerInstanceId,
2390 occurredAt: now,
2391 payload: baseProviderThread,
2392 },
2393 ],
2394 });
2395
2396 const ownedClear = yield* eventSink.writeIfProviderThreadOwner({
2397 providerThreadId,
2398 runId,
2399 activeAttemptId: attemptId,
2400 expectedLastRunOrdinal: 1,
2401 events: [
2402 {
2403 id: EventId.make("event:foundation-provider-thread-owner:clear"),
2404 type: "provider-thread.updated",
2405 threadId,
2406 driver: providerDriver,
2407 providerInstanceId,
2408 occurredAt: now,
2409 payload: {
2410 ...baseProviderThread,
2411 pendingBackgroundTasks: [],
2412 updatedAt: now,
2413 },
2414 },
2415 ],
2416 });
2417 assert.isTrue(ownedClear.committed);
2418 assert.equal(ownedClear.storedEvents.length, 1);
2419
2420 const afterReplacement = yield* DateTime.now;
2421 yield* eventSink.write({
2422 events: [
2423 {
2424 id: EventId.make("event:foundation-provider-thread-owner:replacement-attempt"),
2425 type: "run.updated",
2426 threadId,
2427 runId,
2428 nodeId: rootNodeId,
2429 providerInstanceId,
2430 occurredAt: afterReplacement,
2431 payload: {
2432 ...run,
2433 activeAttemptId: replacementAttemptId,
2434 status: "running",
2435 },
2436 },
2437 ],
2438 });
2439
2440 const supersededAttemptWrite = yield* eventSink.writeIfProviderThreadOwner({
2441 providerThreadId,
2442 runId,
2443 activeAttemptId: attemptId,
2444 expectedLastRunOrdinal: 1,
2445 events: [
2446 {
2447 id: EventId.make("event:foundation-provider-thread-owner:superseded-attempt"),
2448 type: "provider-thread.updated",
2449 threadId,
2450 driver: providerDriver,
2451 providerInstanceId,
2452 occurredAt: afterReplacement,
2453 payload: {
2454 ...baseProviderThread,
2455 status: "active",
2456 pendingBackgroundTasks: [
2457 {
2458 taskId: "bg-superseded",
2459 description: "should not land",
2460 kind: "command" as const,
2461 },
2462 ],
2463 updatedAt: afterReplacement,
2464 },
2465 },
2466 ],
2467 });
2468 assert.isFalse(supersededAttemptWrite.committed);
2469 assert.deepEqual(supersededAttemptWrite.storedEvents, []);
2470
2471 yield* eventSink.write({
2472 events: [
2473 {
2474 id: EventId.make("event:foundation-provider-thread-owner:replacement-completed"),
2475 type: "run.updated",
2476 threadId,
2477 runId,
2478 nodeId: rootNodeId,
2479 providerInstanceId,
2480 occurredAt: afterReplacement,
2481 payload: {
2482 ...run,
2483 activeAttemptId: replacementAttemptId,
2484 },
2485 },
2486 {
2487 id: EventId.make("event:foundation-provider-thread-owner:newer-run"),
2488 type: "provider-thread.updated",
2489 threadId,
2490 driver: providerDriver,
2491 providerInstanceId,
2492 occurredAt: afterReplacement,
2493 payload: {
2494 ...baseProviderThread,
2495 lastRunOrdinal: 2,
2496 status: "active",
2497 pendingBackgroundTasks: [],
2498 updatedAt: afterReplacement,
2499 },
2500 },
2501 ],
2502 });
2503
2504 const staleOrdinalWrite = yield* eventSink.writeIfProviderThreadOwner({
2505 providerThreadId,
2506 runId,
2507 activeAttemptId: replacementAttemptId,
2508 expectedLastRunOrdinal: 1,
2509 events: [
2510 {
2511 id: EventId.make("event:foundation-provider-thread-owner:stale-ordinal"),
2512 type: "provider-thread.updated",
2513 threadId,
2514 driver: providerDriver,
2515 providerInstanceId,
2516 occurredAt: afterReplacement,
2517 payload: baseProviderThread,
2518 },
2519 ],
2520 });
2521 assert.isFalse(staleOrdinalWrite.committed);
2522 assert.deepEqual(staleOrdinalWrite.storedEvents, []);
2523
2524 const projection = yield* projectionStore.getThreadProjection(threadId);
2525 const providerThread = projection.providerThreads.find(
2526 (candidate) => candidate.id === providerThreadId,
2527 );
2528 assert.isDefined(providerThread);
2529 assert.equal(providerThread?.lastRunOrdinal, 2);
2530 assert.equal(providerThread?.status, "active");
2531 assert.deepEqual(providerThread?.pendingBackgroundTasks ?? [], []);
2532 }),
2533 );
2534
2535 it.effect("interrupts a running process-bound effect when it is cancelled", () =>
2536 Effect.gen(function* () {
2537 const outbox = yield* EffectOutbox.EffectOutboxV2;
2538 const commandId = CommandId.make("command:foundation-cancel-running-effect");
2539 const threadId = ThreadId.make("thread:foundation-cancel-running-effect");
2540 const effectId = "effect:foundation-cancel-running-effect";
2541 const started = yield* Deferred.make<void>();
2542 const interrupted = yield* Deferred.make<void>();
2543 yield* outbox.enqueue([
2544 {
2545 id: effectId,
2546 commandId,
2547 threadId,
2548 request: {
2549 type: "provider-turn.start",
2550 runId: RunId.make("run:foundation-cancel-running-effect"),
2551 },
2552 },
2553 ]);
2554
2555 const layerExecutor = Layer.succeed(
2556 EffectWorker.OrchestrationEffectExecutorV2,
2557 EffectWorker.OrchestrationEffectExecutorV2.of({
2558 execute: () =>
2559 Deferred.succeed(started, undefined).pipe(
2560 Effect.andThen(Effect.never),
2561 Effect.onInterrupt(() =>
2562 Deferred.succeed(interrupted, undefined).pipe(Effect.ignore),
2563 ),
2564 ),
2565 }),
2566 );
2567 const layerWorker = EffectWorker.layerWithOptions({
2568 workerId: "cancellation-worker",
2569 }).pipe(
2570 Layer.provide(
2571 Layer.merge(Layer.succeed(EffectOutbox.EffectOutboxV2, outbox), layerExecutor),
2572 ),
2573 );
2574
2575 yield* Effect.gen(function* () {
2576 const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
2577 const workerFiber = yield* worker.runOnce().pipe(Effect.forkChild);
2578 yield* Deferred.await(started);
2579 const cancelledEffectIds = yield* outbox.cancelUnsettled({
2580 threadId,
2581 effectTypes: ["provider-turn.start"],
2582 reason: "The owning run was interrupted.",
2583 });
2584 assert.deepEqual(cancelledEffectIds, [effectId]);
2585 yield* outbox.signalCancellations(cancelledEffectIds);
2586 assert.isTrue(yield* Fiber.join(workerFiber));
2587 yield* Deferred.await(interrupted);
2588 }).pipe(Effect.provide(layerWorker));
2589
2590 const cancelled = yield* outbox.get(effectId);
2591 assert.isTrue(Option.isSome(cancelled));
2592 if (Option.isSome(cancelled)) assert.equal(cancelled.value.status, "cancelled");
2593 }),
2594 );
2595
2596 it.effect("does not resolve awaitSettled waiters until the cancel is signalled", () =>
2597 Effect.gen(function* () {
2598 const outbox = yield* EffectOutbox.EffectOutboxV2;
2599 const commandId = CommandId.make("command:foundation-settle-on-cancel");
2600 const threadId = ThreadId.make("thread:foundation-settle-on-cancel");
2601 const effectId = "effect:foundation-settle-on-cancel";
2602 yield* outbox.enqueue([
2603 {
2604 id: effectId,
2605 commandId,
2606 threadId,
2607 request: {
2608 type: "provider-turn.start",
2609 runId: RunId.make("run:foundation-settle-on-cancel"),
2610 },
2611 },
2612 ]);
2613
2614 const waiterRegistered = yield* Deferred.make<void>();
2615 const waiterFiber = yield* Deferred.succeed(waiterRegistered, undefined).pipe(
2616 Effect.andThen(outbox.awaitSettled(effectId)),
2617 Effect.forkChild,
2618 );
2619 yield* Deferred.await(waiterRegistered);
2620
2621 // Padding enqueues keep the waiter running so its row read lands before
2622 // the cancel: registration is synchronous and the read dispatches first.
2623 // A stronger milestone would need a test-only seam in the service, which
2624 // the codebase avoids.
2625 const paddingCommandId = CommandId.make("command:foundation-settle-on-cancel:padding");
2626 yield* Effect.forEach(
2627 Array.from({ length: 100 }, (_, index) => index),
2628 (index) =>
2629 outbox.enqueue([
2630 {
2631 id: `effect:foundation-settle-on-cancel:padding-${index}`,
2632 commandId: paddingCommandId,
2633 threadId: ThreadId.make("thread:foundation-settle-on-cancel:padding"),
2634 request: { type: "terminal.cleanup" as const },
2635 },
2636 ]),
2637 { discard: true },
2638 );
2639
2640 const cancelledEffectIds = yield* outbox.cancelUnsettled({
2641 threadId,
2642 effectTypes: ["provider-turn.start"],
2643 reason: "Test settle on cancel.",
2644 });
2645 assert.deepEqual(cancelledEffectIds, [effectId]);
2646 // The cancel runs inside the commit transaction; waiters resolve only via
2647 // the post-commit signalCancellations, never from the cancel itself.
2648 yield* Effect.yieldNow;
2649 assert.isUndefined(waiterFiber.pollUnsafe());
2650 yield* outbox.signalCancellations(cancelledEffectIds);
2651 yield* Fiber.join(waiterFiber);
2652 }).pipe(Effect.provide(Layer.fresh(layerEffectOutboxProvided))),
2653 );
2654
2655 it.effect("does not start an effect that was cancelled during claim registration", () =>
2656 Effect.gen(function* () {
2657 const effectId = "effect:foundation-cancelled-during-claim";
2658 const threadId = ThreadId.make("thread:foundation-cancelled-during-claim");
2659 const commandId = CommandId.make("command:foundation-cancelled-during-claim");
2660 const now = DateTime.formatIso(yield* DateTime.now);
2661 const claimedEffect = {
2662 id: effectId,
2663 commandId,
2664 threadId,
2665 request: { type: "terminal.cleanup" as const },
2666 status: "running" as const,
2667 attemptCount: 1,
2668 availableAt: now,
2669 leaseOwner: "claim-cancellation-worker",
2670 createdAt: now,
2671 updatedAt: now,
2672 completedAt: null,
2673 lastError: null,
2674 };
2675 const executionCount = yield* Ref.make(0);
2676 const layerOutbox = Layer.mock(EffectOutbox.EffectOutboxV2)({
2677 claimNext: () => Effect.succeedSome(claimedEffect),
2678 get: () =>
2679 Effect.succeedSome({
2680 ...claimedEffect,
2681 status: "cancelled" as const,
2682 leaseOwner: null,
2683 completedAt: now,
2684 }),
2685 clearCancellation: () => Effect.void,
2686 awaitCancellation: () => Effect.never,
2687 });
2688 const layerExecutor = Layer.succeed(
2689 EffectWorker.OrchestrationEffectExecutorV2,
2690 EffectWorker.OrchestrationEffectExecutorV2.of({
2691 execute: () => Ref.update(executionCount, (count) => count + 1),
2692 }),
2693 );
2694 const layerWorker = EffectWorker.layerWithOptions({
2695 workerId: "claim-cancellation-worker",
2696 }).pipe(Layer.provide(Layer.merge(layerOutbox, layerExecutor)));
2697
2698 assert.isTrue(
2699 yield* EffectWorker.OrchestrationEffectWorkerV2.pipe(
2700 Effect.flatMap((worker) => worker.runOnce()),
2701 Effect.provide(layerWorker),
2702 ),
2703 );
2704 assert.equal(yield* Ref.get(executionCount), 0);
2705 }),
2706 );
2707
2708 it.effect("allows only one worker to claim an available effect", () =>
2709 Effect.gen(function* () {
2710 const outbox = yield* EffectOutbox.EffectOutboxV2;
2711 const commandId = CommandId.make("command:foundation-exclusive-claim");
2712 yield* outbox.enqueue([
2713 {
2714 id: "effect:foundation-exclusive-claim",
2715 commandId,
2716 threadId: ThreadId.make("thread:foundation-exclusive-claim"),
2717 request: {
2718 type: "provider-turn.start",
2719 runId: RunId.make("run:foundation-exclusive-claim"),
2720 },
2721 },
2722 ]);
2723
2724 const claims = yield* Effect.all(
2725 [outbox.claimNext({ workerId: "worker-a" }), outbox.claimNext({ workerId: "worker-b" })],
2726 { concurrency: "unbounded" },
2727 );
2728 assert.equal(claims.filter(Option.isSome).length, 1);
2729 assert.equal(claims.filter(Option.isNone).length, 1);
2730 const claimedByA = claims[0];
2731 const claimedByB = claims[1];
2732 if (Option.isSome(claimedByA)) {
2733 yield* outbox.succeed({ effectId: claimedByA.value.id, workerId: "worker-a" });
2734 }
2735 if (Option.isSome(claimedByB)) {
2736 yield* outbox.succeed({ effectId: claimedByB.value.id, workerId: "worker-b" });
2737 }
2738 }),
2739 );
2740
2741 it.effect("wakes a settle waiter when its effect finishes", () =>
2742 Effect.gen(function* () {
2743 const outbox = yield* EffectOutbox.EffectOutboxV2;
2744 const effectId = "effect:foundation-await-settled";
2745 yield* outbox.enqueue([
2746 {
2747 id: effectId,
2748 commandId: CommandId.make("command:foundation-await-settled"),
2749 threadId: ThreadId.make("thread:foundation-await-settled"),
2750 request: {
2751 type: "provider-runtime.continue",
2752 sourceRunId: RunId.make("run:foundation-await-settled"),
2753 },
2754 },
2755 ]);
2756 const waiter = yield* outbox.awaitSettled(effectId).pipe(Effect.forkChild);
2757 const claimed = yield* outbox.claimNext({
2758 workerId: "settle-worker",
2759 });
2760 assert.isTrue(Option.isSome(claimed));
2761 yield* Effect.yieldNow;
2762 assert.isUndefined(waiter.pollUnsafe());
2763
2764 yield* outbox.succeed({ effectId, workerId: "settle-worker" });
2765 yield* Fiber.join(waiter);
2766 yield* outbox.awaitSettled(effectId);
2767 }),
2768 );
2769
2770 it.effect("runs title generation beside critical work while serializing each lane", () =>
2771 Effect.gen(function* () {
2772 const outbox = yield* EffectOutbox.EffectOutboxV2;
2773 const commandId = CommandId.make("command:foundation-title-effect-lane");
2774 const threadId = ThreadId.make("thread:foundation-title-effect-lane");
2775 const titleEffectId = "effect:foundation-title-effect-lane:a-title";
2776 const providerEffectId = "effect:foundation-title-effect-lane:b-provider";
2777 const nextTitleEffectId = "effect:foundation-title-effect-lane:c-title";
2778 const nextCriticalEffectId = "effect:foundation-title-effect-lane:d-critical";
2779 yield* outbox.enqueue([
2780 {
2781 id: titleEffectId,
2782 commandId,
2783 threadId,
2784 request: {
2785 type: "thread-title.generate",
2786 kind: {
2787 type: "initial",
2788 messageId: MessageId.make("message:foundation-title-effect-lane"),
2789 },
2790 },
2791 },
2792 {
2793 id: providerEffectId,
2794 commandId,
2795 threadId,
2796 request: {
2797 type: "provider-turn.start",
2798 runId: RunId.make("run:foundation-title-effect-lane"),
2799 },
2800 },
2801 {
2802 id: nextTitleEffectId,
2803 commandId,
2804 threadId,
2805 request: { type: "thread-title.generate", kind: { type: "regenerate" } },
2806 },
2807 {
2808 id: nextCriticalEffectId,
2809 commandId,
2810 threadId,
2811 request: { type: "terminal.cleanup" },
2812 },
2813 ]);
2814
2815 const title = yield* outbox.claimNext({
2816 workerId: "title-lane-worker",
2817 });
2818 assert.isTrue(Option.isSome(title));
2819 if (Option.isNone(title)) return;
2820 assert.equal(title.value.id, titleEffectId);
2821
2822 const provider = yield* outbox.claimNext({
2823 workerId: "critical-lane-worker",
2824 });
2825 assert.isTrue(Option.isSome(provider));
2826 if (Option.isNone(provider)) return;
2827 assert.equal(provider.value.id, providerEffectId);
2828
2829 assert.isTrue(
2830 Option.isNone(
2831 yield* outbox.claimNext({
2832 workerId: "blocked-lanes-worker",
2833 }),
2834 ),
2835 );
2836
2837 yield* outbox.succeed({
2838 effectId: provider.value.id,
2839 workerId: "critical-lane-worker",
2840 });
2841 const nextCritical = yield* outbox.claimNext({
2842 workerId: "critical-lane-worker",
2843 });
2844 assert.isTrue(Option.isSome(nextCritical));
2845 if (Option.isNone(nextCritical)) return;
2846 assert.equal(nextCritical.value.id, nextCriticalEffectId);
2847 yield* outbox.succeed({
2848 effectId: nextCritical.value.id,
2849 workerId: "critical-lane-worker",
2850 });
2851
2852 yield* outbox.succeed({
2853 effectId: title.value.id,
2854 workerId: "title-lane-worker",
2855 });
2856 const nextTitle = yield* outbox.claimNext({
2857 workerId: "title-lane-worker",
2858 });
2859 assert.isTrue(Option.isSome(nextTitle));
2860 if (Option.isSome(nextTitle)) {
2861 assert.equal(nextTitle.value.id, nextTitleEffectId);
2862 yield* outbox.succeed({
2863 effectId: nextTitle.value.id,
2864 workerId: "title-lane-worker",
2865 });
2866 }
2867 }),
2868 );
2869
2870 it.effect("ignores deadlines blocked by a running effect on the same thread", () =>
2871 Effect.gen(function* () {
2872 const outbox = yield* EffectOutbox.EffectOutboxV2;
2873 const now = yield* DateTime.now;
2874 const future = DateTime.add(now, { seconds: 10 });
2875 const commandId = CommandId.make("command:foundation-next-claimable");
2876 const blockedThreadId = ThreadId.make("thread:foundation-next-claimable:blocked");
2877 yield* outbox.enqueue([
2878 {
2879 id: "effect:foundation-next-claimable:a1",
2880 commandId,
2881 threadId: blockedThreadId,
2882 request: { type: "terminal.cleanup" },
2883 },
2884 {
2885 id: "effect:foundation-next-claimable:a2",
2886 commandId,
2887 threadId: blockedThreadId,
2888 request: { type: "terminal.cleanup" },
2889 },
2890 {
2891 id: "effect:foundation-next-claimable:b1",
2892 commandId,
2893 threadId: ThreadId.make("thread:foundation-next-claimable:future"),
2894 request: { type: "terminal.cleanup" },
2895 availableAt: future,
2896 },
2897 ]);
2898
2899 const claimed = yield* outbox.claimNext({
2900 workerId: "next-claimable-worker",
2901 });
2902 assert.isTrue(Option.isSome(claimed));
2903 if (Option.isNone(claimed)) return;
2904 assert.equal(claimed.value.id, "effect:foundation-next-claimable:a1");
2905
2906 const whileBlocked = yield* outbox.nextClaimableAt();
2907 assert.isTrue(Option.isSome(whileBlocked));
2908 if (Option.isSome(whileBlocked)) {
2909 assert.equal(DateTime.toEpochMillis(whileBlocked.value), DateTime.toEpochMillis(future));
2910 }
2911
2912 yield* outbox.succeed({
2913 effectId: claimed.value.id,
2914 workerId: "next-claimable-worker",
2915 });
2916 const afterCompletion = yield* outbox.nextClaimableAt();
2917 assert.isTrue(Option.isSome(afterCompletion));
2918 if (Option.isSome(afterCompletion)) {
2919 assert.isAtMost(DateTime.toEpochMillis(afterCompletion.value), DateTime.toEpochMillis(now));
2920 }
2921
2922 const unblocked = yield* outbox.claimNext({
2923 workerId: "next-claimable-worker",
2924 });
2925 assert.isTrue(Option.isSome(unblocked));
2926 if (Option.isSome(unblocked)) {
2927 assert.equal(unblocked.value.id, "effect:foundation-next-claimable:a2");
2928 yield* outbox.succeed({
2929 effectId: unblocked.value.id,
2930 workerId: "next-claimable-worker",
2931 });
2932 }
2933 yield* outbox.cancelUnsettled({
2934 threadId: ThreadId.make("thread:foundation-next-claimable:future"),
2935 effectTypes: ["terminal.cleanup"],
2936 reason: "Test cleanup.",
2937 });
2938 }),
2939 );
2940
2941 it.effect("prunes settled effects past retention in batches and keeps the rest", () =>
2942 Effect.gen(function* () {
2943 const outbox = yield* EffectOutbox.EffectOutboxV2;
2944 const sql = yield* SqlClient.SqlClient;
2945 const now = yield* DateTime.now;
2946 const insert = (
2947 prefix: string,
2948 count: number,
2949 status: EffectOutbox.OrchestrationEffectStatusV2,
2950 completedAgo: Duration.Duration | null,
2951 ) => {
2952 const completedAt =
2953 completedAgo === null
2954 ? null
2955 : DateTime.formatIso(DateTime.subtractDuration(now, completedAgo));
2956 const createdAt = DateTime.formatIso(now);
2957 return sql`
2958 WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < ${count})
2959 INSERT INTO orchestration_v2_effect_outbox (
2960 effect_id, command_id, thread_id, effect_type, payload_json, status,
2961 available_at, created_at, updated_at, completed_at
2962 )
2963 SELECT ${prefix} || i, 'command:prune', 'thread:prune', 'terminal.cleanup',
2964 '{"type":"terminal.cleanup"}', ${status}, ${createdAt}, ${createdAt}, ${createdAt},
2965 ${completedAt}
2966 FROM n
2967 `;
2968 };
2969 const old = Duration.sum(EffectOutbox.SETTLED_EFFECT_RETENTION, Duration.minutes(1));
2970 const recent = Duration.subtract(EffectOutbox.SETTLED_EFFECT_RETENTION, Duration.minutes(1));
2971 yield* insert("succeeded-old:", 1_201, "succeeded", old);
2972 yield* insert("cancelled-old:", 2, "cancelled", old);
2973 yield* insert("succeeded-recent:", 1, "succeeded", recent);
2974 yield* insert("failed-old:", 1, "failed", old);
2975 yield* insert("pending:", 1, "pending", null);
2976 yield* insert("running:", 1, "running", null);
2977
2978 assert.equal(yield* outbox.pruneSettled, 1_203);
2979
2980 const remaining = yield* sql<{ readonly effect_id: string }>`
2981 SELECT effect_id FROM orchestration_v2_effect_outbox ORDER BY effect_id
2982 `;
2983 assert.deepEqual(
2984 remaining.map((row) => row.effect_id),
2985 ["failed-old:1", "pending:1", "running:1", "succeeded-recent:1"],
2986 );
2987 assert.equal(yield* outbox.pruneSettled, 0);
2988 }).pipe(Effect.provide(layerIsolatedOutbox)),
2989 );
2990
2991 it.effect("fails an effect this build cannot decode and claims the thread's next effect", () =>
2992 Effect.gen(function* () {
2993 const outbox = yield* EffectOutbox.EffectOutboxV2;
2994 const sql = yield* SqlClient.SqlClient;
2995 const nowIso = DateTime.formatIso(yield* DateTime.now);
2996 const workerId = "undecodable-worker";
2997 const commandId = CommandId.make("command:foundation-undecodable");
2998 const threadId = ThreadId.make("thread:foundation-undecodable");
2999 yield* sql`
3000 INSERT INTO orchestration_v2_effect_outbox (
3001 effect_id, command_id, thread_id, effect_type, payload_json, status,
3002 available_at, created_at, updated_at
3003 )
3004 VALUES (
3005 'effect:foundation-undecodable:a', ${commandId}, ${threadId}, 'future-effect.kind',
3006 '{"type":"future-effect.kind"}', 'pending', ${nowIso}, ${nowIso}, ${nowIso}
3007 )
3008 `;
3009 yield* outbox.enqueue([
3010 {
3011 id: "effect:foundation-undecodable:b",
3012 commandId,
3013 threadId,
3014 request: { type: "provider-turn.start", runId: RunId.make("run:foundation-undecodable") },
3015 },
3016 ]);
3017
3018 const claimed = yield* Effect.exit(outbox.claimNext({ workerId }));
3019
3020 const rows = yield* sql<{
3021 readonly effect_id: string;
3022 readonly status: string;
3023 readonly lease_owner: string | null;
3024 readonly last_error: string | null;
3025 }>`
3026 SELECT effect_id, status, lease_owner, last_error
3027 FROM orchestration_v2_effect_outbox
3028 ORDER BY effect_id
3029 `;
3030 assert.deepEqual(rows, [
3031 {
3032 effect_id: "effect:foundation-undecodable:a",
3033 status: "failed",
3034 lease_owner: null,
3035 last_error: "This server build cannot decode the future-effect.kind effect.",
3036 },
3037 {
3038 effect_id: "effect:foundation-undecodable:b",
3039 status: "running",
3040 lease_owner: workerId,
3041 last_error: null,
3042 },
3043 ]);
3044 assert.equal(
3045 Exit.isSuccess(claimed) ? Option.getOrUndefined(claimed.value)?.id : undefined,
3046 "effect:foundation-undecodable:b",
3047 );
3048 }).pipe(Effect.provide(layerIsolatedOutbox)),
3049 );
3050
3051 it.effect.each(["queued", "waiting"])(
3052 "keeps a %s continuation's crash budget past outbox retention",
3053 (status) =>
3054 Effect.gen(function* () {
3055 const outbox = yield* EffectOutbox.EffectOutboxV2;
3056 const sql = yield* SqlClient.SqlClient;
3057 const now = yield* DateTime.now;
3058 const old = DateTime.formatIso(DateTime.subtractDuration(now, Duration.days(8)));
3059 const timestamp = DateTime.formatIso(now);
3060 yield* sql`
3061 INSERT INTO orchestration_v2_effect_outbox
3062 (effect_id, command_id, thread_id, effect_type, payload_json, status, available_at, created_at, updated_at, completed_at)
3063 VALUES ('effect:retained-recovery', 'command:retained-recovery', 'thread:retained-recovery',
3064 'provider-runtime.continue', '{"type":"provider-runtime.continue","sourceRunId":"run:retained-source","processLossCount":3}',
3065 'succeeded', ${old}, ${old}, ${old}, ${old})
3066 `;
3067 yield* sql`
3068 INSERT INTO orchestration_v2_projection_runs
3069 (run_id, thread_id, ordinal, provider, status, requested_at, payload_json)
3070 VALUES ('run:retained-recovery', 'thread:retained-recovery', 1, 'codex', ${status}, ${timestamp},
3071 '{"restartContinuationOfRunId":"run:retained-source"}')
3072 `;
3073 assert.equal(yield* outbox.pruneSettled, 0);
3074 yield* sql`UPDATE orchestration_v2_projection_runs SET status = 'completed' WHERE run_id = 'run:retained-recovery'`;
3075 assert.equal(yield* outbox.pruneSettled, 1);
3076 }).pipe(Effect.provide(layerIsolatedOutbox)),
3077 );
3078
3079 it.effect("wakes claimers when cancellation unblocks same-thread work", () =>
3080 Effect.gen(function* () {
3081 const outbox = yield* EffectOutbox.EffectOutboxV2;
3082 const commandId = CommandId.make("command:foundation-cancellation-wakeup");
3083 const threadId = ThreadId.make("thread:foundation-cancellation-wakeup");
3084 yield* outbox.enqueue([
3085 {
3086 id: "effect:foundation-cancellation-wakeup:a-running",
3087 commandId,
3088 threadId,
3089 request: {
3090 type: "provider-turn.start",
3091 runId: RunId.make("run:foundation-cancellation-wakeup"),
3092 },
3093 },
3094 {
3095 id: "effect:foundation-cancellation-wakeup:b-pending",
3096 commandId,
3097 threadId,
3098 request: { type: "terminal.cleanup" },
3099 },
3100 ]);
3101
3102 const running = yield* outbox.claimNext({
3103 workerId: "cancellation-wakeup-worker",
3104 });
3105 assert.isTrue(Option.isSome(running));
3106 if (Option.isNone(running)) return;
3107 assert.equal(running.value.request.type, "provider-turn.start");
3108
3109 const cancelledEffectIds = yield* outbox.cancelUnsettled({
3110 threadId,
3111 effectTypes: ["provider-turn.start"],
3112 reason: "Test cancellation wakeup.",
3113 });
3114 yield* outbox.signalCancellations(cancelledEffectIds);
3115
3116 const wake = yield* outbox.awaitAvailable.pipe(Effect.forkChild);
3117 yield* Effect.yieldNow;
3118 assert.isDefined(wake.pollUnsafe());
3119
3120 const unblocked = yield* outbox.claimNext({
3121 workerId: "cancellation-wakeup-worker",
3122 });
3123 assert.isTrue(Option.isSome(unblocked));
3124 if (Option.isSome(unblocked)) {
3125 assert.equal(unblocked.value.request.type, "terminal.cleanup");
3126 yield* outbox.succeed({
3127 effectId: unblocked.value.id,
3128 workerId: "cancellation-wakeup-worker",
3129 });
3130 }
3131 }).pipe(Effect.provide(Layer.fresh(layerEffectOutboxProvided))),
3132 );
3133
3134 it.effect("keeps later thread effects behind an earlier effect waiting to retry", () =>
3135 Effect.gen(function* () {
3136 const outbox = yield* EffectOutbox.EffectOutboxV2;
3137 const workerId = "retry-order-worker";
3138 const commandId = CommandId.make("command:foundation-retry-order");
3139 const threadId = ThreadId.make("thread:foundation-retry-order");
3140 yield* outbox.enqueue([
3141 {
3142 id: "effect:foundation-retry-order:z-rollback",
3143 commandId,
3144 threadId,
3145 request: {
3146 type: "provider-thread.rollback",
3147 providerThreadId: ProviderThreadId.make("provider-thread:foundation-retry-order"),
3148 checkpointId: CheckpointId.make("checkpoint:foundation-retry-order"),
3149 scopeId: CheckpointScopeId.make("scope:foundation-retry-order"),
3150 },
3151 },
3152 ]);
3153 const rollback = yield* outbox.claimNext({ workerId });
3154 assert.isTrue(Option.isSome(rollback));
3155 if (Option.isNone(rollback)) return;
3156 yield* outbox.retry({
3157 effectId: rollback.value.id,
3158 workerId,
3159 error: "rollback failed once",
3160 delayMs: 60_000,
3161 });
3162
3163 yield* outbox.enqueue([
3164 {
3165 id: "effect:foundation-retry-order:a-start",
3166 commandId: CommandId.make("command:foundation-retry-order:start"),
3167 threadId,
3168 request: { type: "provider-turn.start", runId: RunId.make("run:foundation-retry-order") },
3169 },
3170 {
3171 id: "effect:foundation-retry-order:b-title",
3172 commandId: CommandId.make("command:foundation-retry-order:title"),
3173 threadId,
3174 request: { type: "thread-title.generate", kind: { type: "regenerate" } },
3175 },
3176 ]);
3177 const title = yield* outbox.claimNext({ workerId });
3178 assert.equal(Option.getOrUndefined(title)?.id, "effect:foundation-retry-order:b-title");
3179 const blocked = yield* outbox.claimNext({ workerId });
3180 assert.isTrue(Option.isNone(blocked));
3181 const nextClaimable = yield* outbox.nextClaimableAt();
3182 assert.isTrue(Option.isSome(nextClaimable));
3183 if (Option.isSome(nextClaimable)) {
3184 assert.equal(
3185 DateTime.formatIso(nextClaimable.value),
3186 (yield* outbox.get(rollback.value.id)).pipe(Option.getOrThrow).availableAt,
3187 );
3188 }
3189
3190 yield* outbox.cancelUnsettled({
3191 threadId,
3192 effectTypes: ["provider-thread.rollback"],
3193 reason: "Test cleanup.",
3194 });
3195 const unblocked = yield* outbox.claimNext({ workerId });
3196 assert.equal(Option.getOrUndefined(unblocked)?.id, "effect:foundation-retry-order:a-start");
3197 yield* outbox.succeed({ effectId: "effect:foundation-retry-order:a-start", workerId });
3198 yield* outbox.succeed({ effectId: "effect:foundation-retry-order:b-title", workerId });
3199 }).pipe(Effect.provide(Layer.fresh(layerEffectOutboxProvided))),
3200 );
3201
3202 it.effect("runs distinct threads concurrently while serializing effects within a thread", () =>
3203 Effect.gen(function* () {
3204 const outbox = yield* EffectOutbox.EffectOutboxV2;
3205 const commandId = CommandId.make("command:foundation-concurrent-effects");
3206 const threadA = ThreadId.make("thread:foundation-concurrent-effects:a");
3207 const threadB = ThreadId.make("thread:foundation-concurrent-effects:b");
3208 const effectA1 = "effect:foundation-concurrent-effects:a1";
3209 const effectA2 = "effect:foundation-concurrent-effects:a2";
3210 const effectB1 = "effect:foundation-concurrent-effects:b1";
3211 const startedA1 = yield* Deferred.make<void>();
3212 const startedA2 = yield* Deferred.make<void>();
3213 const startedB1 = yield* Deferred.make<void>();
3214 const releaseA1 = yield* Deferred.make<void>();
3215 const releaseA2 = yield* Deferred.make<void>();
3216 const releaseB1 = yield* Deferred.make<void>();
3217 const settled = yield* Deferred.make<void>();
3218 const completed = yield* Ref.make(0);
3219 const gates = new Map([
3220 [effectA1, { started: startedA1, release: releaseA1 }],
3221 [effectA2, { started: startedA2, release: releaseA2 }],
3222 [effectB1, { started: startedB1, release: releaseB1 }],
3223 ]);
3224 const layerExecutor = Layer.succeed(
3225 EffectWorker.OrchestrationEffectExecutorV2,
3226 EffectWorker.OrchestrationEffectExecutorV2.of({
3227 execute: (effect) => {
3228 const gate = gates.get(effect.id);
3229 if (gate === undefined) return Effect.die(`Missing gate for ${effect.id}`);
3230 return Deferred.succeed(gate.started, undefined).pipe(
3231 Effect.andThen(Deferred.await(gate.release)),
3232 );
3233 },
3234 }),
3235 );
3236 const layerWorker = EffectWorker.layerWithOptions({
3237 workerId: "concurrency-worker",
3238 }).pipe(
3239 Layer.provide(
3240 Layer.merge(
3241 Layer.succeed(EffectOutbox.EffectOutboxV2, {
3242 ...outbox,
3243 succeed: (input) =>
3244 outbox
3245 .succeed(input)
3246 .pipe(
3247 Effect.tap(() =>
3248 Ref.updateAndGet(completed, (count) => count + 1).pipe(
3249 Effect.flatMap((count) =>
3250 count === 3 ? Deferred.succeed(settled, undefined) : Effect.void,
3251 ),
3252 ),
3253 ),
3254 ),
3255 }),
3256 layerExecutor,
3257 ),
3258 ),
3259 );
3260
3261 yield* Effect.gen(function* () {
3262 yield* EffectWorker.runDaemonWithOptions({ concurrency: 2 }).pipe(Effect.forkScoped);
3263 yield* Effect.yieldNow;
3264 yield* Effect.yieldNow;
3265 yield* outbox.enqueue([
3266 {
3267 id: effectA1,
3268 commandId,
3269 threadId: threadA,
3270 request: {
3271 type: "provider-turn.start",
3272 runId: RunId.make("run:foundation-concurrent-effects:a1"),
3273 },
3274 },
3275 {
3276 id: effectA2,
3277 commandId,
3278 threadId: threadA,
3279 request: {
3280 type: "provider-turn.start",
3281 runId: RunId.make("run:foundation-concurrent-effects:a2"),
3282 },
3283 },
3284 {
3285 id: effectB1,
3286 commandId,
3287 threadId: threadB,
3288 request: {
3289 type: "provider-turn.start",
3290 runId: RunId.make("run:foundation-concurrent-effects:b1"),
3291 },
3292 },
3293 ]);
3294 yield* outbox.notifyAvailable(3);
3295
3296 yield* Effect.all([Deferred.await(startedA1), Deferred.await(startedB1)]);
3297 assert.isFalse(yield* Deferred.isDone(startedA2));
3298
3299 yield* Deferred.succeed(releaseA1, undefined);
3300 yield* Deferred.succeed(releaseB1, undefined);
3301 yield* Deferred.await(startedA2);
3302 yield* Deferred.succeed(releaseA2, undefined);
3303 yield* Deferred.await(settled);
3304 assert.isTrue(
3305 (yield* outbox.listByCommandId(commandId)).every(
3306 (effect) => effect.status === "succeeded",
3307 ),
3308 );
3309 }).pipe(Effect.provide(layerWorker), Effect.scoped);
3310 }),
3311 );
3312
3313 it.effect("starts a provider turn while slow title generations are running", () =>
3314 Effect.gen(function* () {
3315 const outbox = yield* EffectOutbox.EffectOutboxV2;
3316 const commandId = CommandId.make("command:foundation-title-starvation");
3317 const titleThreads = ["a", "b", "c"].map((suffix) =>
3318 ThreadId.make(`thread:foundation-title-starvation:${suffix}`),
3319 );
3320 const turnStarted = yield* Deferred.make<void>();
3321 const releaseTitles = yield* Deferred.make<void>();
3322 const layerWorker = EffectWorker.layerWithOptions({
3323 workerId: "title-starvation-worker",
3324 }).pipe(
3325 Layer.provide(
3326 Layer.merge(
3327 Layer.succeed(EffectOutbox.EffectOutboxV2, outbox),
3328 Layer.succeed(
3329 EffectWorker.OrchestrationEffectExecutorV2,
3330 EffectWorker.OrchestrationEffectExecutorV2.of({
3331 execute: (effect) =>
3332 effect.request.type === "thread-title.generate"
3333 ? Deferred.await(releaseTitles)
3334 : Deferred.succeed(turnStarted, undefined).pipe(Effect.asVoid),
3335 }),
3336 ),
3337 ),
3338 ),
3339 );
3340
3341 yield* Effect.gen(function* () {
3342 yield* EffectWorker.runDaemonWithOptions({ concurrency: 2 }).pipe(Effect.forkScoped);
3343 yield* Effect.yieldNow;
3344 yield* outbox.enqueue([
3345 ...titleThreads.map((threadId, index) => ({
3346 id: `effect:foundation-title-starvation:${index}-title`,
3347 commandId,
3348 threadId,
3349 request: {
3350 type: "thread-title.generate" as const,
3351 kind: { type: "regenerate" as const },
3352 },
3353 })),
3354 {
3355 id: "effect:foundation-title-starvation:turn-start",
3356 commandId,
3357 threadId: titleThreads[0]!,
3358 request: {
3359 type: "provider-turn.start",
3360 runId: RunId.make("run:foundation-title-starvation"),
3361 },
3362 },
3363 ]);
3364 yield* outbox.notifyAvailable(titleThreads.length + 1);
3365
3366 const started = yield* Deferred.await(turnStarted).pipe(
3367 Effect.timeoutOption(Duration.seconds(1)),
3368 Effect.forkScoped,
3369 );
3370 yield* TestClock.adjust(Duration.seconds(1));
3371 assert.isTrue(
3372 Option.isSome(yield* Fiber.join(started)),
3373 "provider-turn.start was never claimed: thread title generations occupied every effect worker",
3374 );
3375 assert.isFalse(yield* Deferred.isDone(releaseTitles));
3376
3377 yield* Deferred.succeed(releaseTitles, undefined);
3378 yield* Effect.forEach(titleThreads, (_, index) =>
3379 outbox.awaitSettled(`effect:foundation-title-starvation:${index}-title`),
3380 );
3381 }).pipe(Effect.provide(layerWorker), Effect.scoped);
3382 }),
3383 );
3384
3385 it.effect("holds pending titles at the running-title cap without waking idle workers", () =>
3386 Effect.gen(function* () {
3387 const outbox = yield* EffectOutbox.EffectOutboxV2;
3388 const cap = { maxRunningTitleGenerations: 1 };
3389 const title = (name: string) => ({
3390 id: `effect:foundation-title-cap:${name}`,
3391 commandId: CommandId.make(`command:foundation-title-cap:${name}`),
3392 threadId: ThreadId.make(`thread:foundation-title-cap:${name}`),
3393 request: { type: "thread-title.generate" as const, kind: { type: "regenerate" as const } },
3394 });
3395 yield* outbox.enqueue([title("first"), title("second")]);
3396 const first = yield* outbox.claimNext({ workerId: "title-cap-worker", ...cap });
3397 assert.strictEqual(Option.getOrUndefined(first)?.id, title("first").id);
3398
3399 assert.isTrue(
3400 Option.isNone(yield* outbox.claimNext({ workerId: "title-cap-worker", ...cap })),
3401 );
3402 assert.isTrue(
3403 Option.isNone(yield* outbox.nextClaimableAt(cap)),
3404 "a capped title must not report a past deadline, or idle workers re-poll every 25 ms",
3405 );
3406 assert.isTrue(Option.isSome(yield* outbox.nextClaimableAt()));
3407
3408 yield* outbox.succeed({ effectId: title("first").id, workerId: "title-cap-worker" });
3409 const second = yield* outbox.claimNext({ workerId: "title-cap-worker", ...cap });
3410 assert.strictEqual(Option.getOrUndefined(second)?.id, title("second").id);
3411 yield* outbox.succeed({ effectId: title("second").id, workerId: "title-cap-worker" });
3412 }),
3413 );
3414
3415 it.effect("does not reclaim a running effect after its process-local lease expires", () =>
3416 Effect.gen(function* () {
3417 const outbox = yield* EffectOutbox.EffectOutboxV2;
3418 const sql = yield* SqlClient.SqlClient;
3419 const commandId = CommandId.make("command:foundation-no-live-reclaim");
3420 const threadId = ThreadId.make("thread:foundation-no-live-reclaim");
3421 const firstEffectId = "effect:foundation-no-live-reclaim:first";
3422 const secondEffectId = "effect:foundation-no-live-reclaim:second";
3423 const firstStarted = yield* Deferred.make<void>();
3424 const releaseFirst = yield* Deferred.make<void>();
3425 const executions = yield* Ref.make<ReadonlyArray<string>>([]);
3426 yield* outbox.enqueue([
3427 {
3428 id: firstEffectId,
3429 commandId,
3430 threadId,
3431 request: { type: "terminal.cleanup" },
3432 },
3433 {
3434 id: secondEffectId,
3435 commandId,
3436 threadId,
3437 request: { type: "terminal.cleanup" },
3438 },
3439 ]);
3440
3441 const layerExecutor = Layer.succeed(
3442 EffectWorker.OrchestrationEffectExecutorV2,
3443 EffectWorker.OrchestrationEffectExecutorV2.of({
3444 execute: (effect) =>
3445 Ref.update(executions, (current) => [...current, effect.id]).pipe(
3446 Effect.andThen(
3447 effect.id === firstEffectId
3448 ? Deferred.succeed(firstStarted, undefined).pipe(
3449 Effect.andThen(Deferred.await(releaseFirst)),
3450 )
3451 : Effect.void,
3452 ),
3453 ),
3454 }),
3455 );
3456 const layerWorker = EffectWorker.layerWithOptions({
3457 workerId: "no-live-reclaim-worker",
3458 }).pipe(
3459 Layer.provide(
3460 Layer.merge(Layer.succeed(EffectOutbox.EffectOutboxV2, outbox), layerExecutor),
3461 ),
3462 );
3463
3464 yield* Effect.gen(function* () {
3465 const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
3466 const firstFiber = yield* worker.runOnce().pipe(Effect.forkChild);
3467 yield* Deferred.await(firstStarted);
3468 yield* sql`
3469 UPDATE orchestration_v2_effect_outbox
3470 SET lease_expires_at = '1970-01-01T00:00:00.000Z'
3471 WHERE effect_id = ${firstEffectId}
3472 `;
3473
3474 assert.isFalse(yield* worker.runOnce());
3475 assert.deepEqual(yield* Ref.get(executions), [firstEffectId]);
3476
3477 yield* Deferred.succeed(releaseFirst, undefined);
3478 assert.isTrue(yield* Fiber.join(firstFiber));
3479 assert.isTrue(yield* worker.runOnce());
3480 assert.deepEqual(yield* Ref.get(executions), [firstEffectId, secondEffectId]);
3481 }).pipe(Effect.provide(layerWorker));
3482 }),
3483 );
3484
3485 it.effect.each([{ replayed: false }, { replayed: true }] as const)(
3486 "starts later thread work behind a delegated-task stop that never returns (replayed: $replayed)",
3487 ({ replayed }) =>
3488 Effect.gen(function* () {
3489 const outbox = yield* EffectOutbox.EffectOutboxV2;
3490 const key = `foundation-hung-delegated-stop:${replayed}`;
3491 const threadId = ThreadId.make(`thread:${key}`);
3492 const stopEffectId = `effect:${key}:stop`;
3493 const stopStarted = yield* Deferred.make<void>();
3494 const turnStarted = yield* Deferred.make<void>();
3495 yield* outbox.enqueue([
3496 {
3497 id: stopEffectId,
3498 commandId: CommandId.make(`command:${key}:stop`),
3499 threadId,
3500 request: { type: "delegated-tasks.stop" },
3501 },
3502 ]);
3503 if (replayed) {
3504 assert.isTrue(Option.isSome(yield* outbox.claimNext({ workerId: "crashed-worker" })));
3505 assert.equal((yield* outbox.reconcileAfterProcessLoss).requeued, 1);
3506 }
3507
3508 const layerExecutor = Layer.succeed(
3509 EffectWorker.OrchestrationEffectExecutorV2,
3510 EffectWorker.OrchestrationEffectExecutorV2.of({
3511 execute: (effect) =>
3512 effect.request.type === "delegated-tasks.stop"
3513 ? Deferred.succeed(stopStarted, undefined).pipe(Effect.andThen(Effect.never))
3514 : Deferred.succeed(turnStarted, undefined),
3515 }),
3516 );
3517 const layerWorker = EffectWorker.layerWithOptions({
3518 workerId: "hung-delegated-stop-worker",
3519 }).pipe(
3520 Layer.provide(
3521 Layer.merge(Layer.succeed(EffectOutbox.EffectOutboxV2, outbox), layerExecutor),
3522 ),
3523 );
3524
3525 yield* Effect.gen(function* () {
3526 yield* EffectWorker.runDaemonWithOptions({ concurrency: 2 }).pipe(Effect.forkScoped);
3527 yield* Deferred.await(stopStarted);
3528 yield* outbox.enqueue([
3529 {
3530 id: `effect:${key}:turn`,
3531 commandId: CommandId.make(`command:${key}:turn`),
3532 threadId,
3533 request: { type: "provider-turn.start", runId: RunId.make(`run:${key}`) },
3534 },
3535 ]);
3536 yield* outbox.notifyAvailable();
3537
3538 const turn = yield* Deferred.await(turnStarted).pipe(
3539 Effect.timeoutOption("1 hour"),
3540 Effect.forkChild,
3541 );
3542 yield* TestClock.adjust("1 hour");
3543 assert.isTrue(
3544 Option.isSome(yield* Fiber.join(turn)),
3545 "provider-turn.start never ran behind the hung delegated-tasks.stop",
3546 );
3547 const stop = yield* outbox.get(stopEffectId);
3548 assert.isTrue(Option.isSome(stop));
3549 if (Option.isSome(stop)) {
3550 assert.equal(stop.value.status, "failed");
3551 assert.include(stop.value.lastError ?? "", "Timed out");
3552 }
3553 }).pipe(
3554 Effect.provide(layerWorker),
3555 Effect.scoped,
3556 Effect.ensuring(
3557 outbox
3558 .cancelUnsettled({
3559 threadId,
3560 effectTypes: ["delegated-tasks.stop", "provider-turn.start"],
3561 reason: "fixture complete",
3562 })
3563 .pipe(Effect.orDie),
3564 ),
3565 );
3566 }),
3567 );
3568
3569 it.effect(
3570 "starts another thread's turn while every worker holds a start that never returns",
3571 () =>
3572 Effect.gen(function* () {
3573 const outbox = yield* EffectOutbox.EffectOutboxV2;
3574 const key = "foundation-hung-provider-starts";
3575 const wedgedThreadIds = [0, 1].map((index) =>
3576 ThreadId.make(`thread:${key}:wedged:${index}`),
3577 );
3578 const otherThreadId = ThreadId.make(`thread:${key}:other`);
3579 const wedgedStarted = yield* Deferred.make<void>();
3580 const otherStarted = yield* Deferred.make<void>();
3581 const wedgedCount = yield* Ref.make(0);
3582 yield* outbox.enqueue(
3583 wedgedThreadIds.map((threadId, index) => ({
3584 id: `effect:${key}:wedged:${index}`,
3585 commandId: CommandId.make(`command:${key}:wedged:${index}`),
3586 threadId,
3587 request: {
3588 type: "provider-turn.start",
3589 runId: RunId.make(`run:${key}:wedged:${index}`),
3590 },
3591 })),
3592 );
3593
3594 const layerExecutor = Layer.succeed(
3595 EffectWorker.OrchestrationEffectExecutorV2,
3596 EffectWorker.OrchestrationEffectExecutorV2.of({
3597 execute: (effect) =>
3598 effect.threadId === otherThreadId
3599 ? Deferred.succeed(otherStarted, undefined)
3600 : Ref.updateAndGet(wedgedCount, (count) => count + 1).pipe(
3601 Effect.flatMap((count) =>
3602 count === wedgedThreadIds.length
3603 ? Deferred.succeed(wedgedStarted, undefined)
3604 : Effect.void,
3605 ),
3606 Effect.andThen(Effect.never),
3607 ),
3608 }),
3609 );
3610 const layerWorker = EffectWorker.layerWithOptions({
3611 workerId: "hung-provider-starts-worker",
3612 }).pipe(
3613 Layer.provide(
3614 Layer.merge(Layer.succeed(EffectOutbox.EffectOutboxV2, outbox), layerExecutor),
3615 ),
3616 );
3617
3618 yield* Effect.gen(function* () {
3619 yield* EffectWorker.runDaemonWithOptions({ concurrency: wedgedThreadIds.length }).pipe(
3620 Effect.forkScoped,
3621 );
3622 yield* Deferred.await(wedgedStarted);
3623 yield* outbox.enqueue([
3624 {
3625 id: `effect:${key}:other`,
3626 commandId: CommandId.make(`command:${key}:other`),
3627 threadId: otherThreadId,
3628 request: { type: "provider-turn.start", runId: RunId.make(`run:${key}:other`) },
3629 },
3630 ]);
3631 yield* outbox.notifyAvailable();
3632
3633 const other = yield* Deferred.await(otherStarted).pipe(
3634 Effect.timeoutOption("1 hour"),
3635 Effect.forkChild,
3636 );
3637 yield* TestClock.adjust("1 hour");
3638 assert.isTrue(
3639 Option.isSome(yield* Fiber.join(other)),
3640 "another thread's provider-turn.start never ran behind starts that never return",
3641 );
3642 }).pipe(
3643 Effect.provide(layerWorker),
3644 Effect.scoped,
3645 Effect.ensuring(
3646 Effect.forEach(
3647 [...wedgedThreadIds, otherThreadId],
3648 (threadId) =>
3649 outbox.cancelUnsettled({
3650 threadId,
3651 effectTypes: ["provider-turn.start"],
3652 reason: "fixture complete",
3653 }),
3654 { discard: true },
3655 ).pipe(Effect.orDie),
3656 ),
3657 );
3658 }),
3659 );
3660
3661 it.effect(
3662 "persists shutdown continuation intent through the real event sink without domain events",
3663 () =>
3664 Effect.gen(function* () {
3665 const outbox = yield* EffectOutbox.EffectOutboxV2;
3666 const now = yield* DateTime.now;
3667 const threadId = ThreadId.make("thread:shutdown-prepare");
3668 const runId = RunId.make("run:shutdown-prepare");
3669 const providerThreadId = ProviderThreadId.make("provider-thread:shutdown-prepare");
3670 const sessionId = ProviderSessionId.make("session:shutdown-prepare");
3671 const attemptId = RunAttemptId.make("attempt:shutdown-prepare");
3672 const projection = {
3673 thread: makeThread(threadId, now),
3674 runs: [
3675 {
3676 id: runId,
3677 ordinal: 1,
3678 status: "running",
3679 providerInstanceId,
3680 providerThreadId,
3681 activeAttemptId: attemptId,
3682 },
3683 ],
3684 providerThreads: [
3685 {
3686 id: providerThreadId,
3687 appThreadId: threadId,
3688 ownerNodeId: null,
3689 driver: "codex",
3690 providerInstanceId,
3691 providerSessionId: sessionId,
3692 status: "active",
3693 nativeThreadRef: {
3694 driver: "codex",
3695 nativeId: "saved-native-thread",
3696 strength: "strong",
3697 },
3698 },
3699 ],
3700 providerSessions: [
3701 { id: sessionId, driver: "codex", providerInstanceId, status: "running" },
3702 ],
3703 providerTurns: [{ providerThreadId, runAttemptId: attemptId, status: "running" }],
3704 runtimeRequests: [],
3705 turnItems: [],
3706 } as unknown as OrchestrationV2ThreadProjection;
3707 const recovery = yield* ProviderRuntimeRecovery.make.pipe(
3708 Effect.provide(
3709 Layer.mergeAll(
3710 ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
3711 Layer.mock(ProjectionStore.ProjectionStoreV2)({
3712 getRecoveryThreadIds: () => Effect.succeed([threadId]),
3713 getRuntimeRecoveryProjection: () => Effect.succeed(projection),
3714 }),
3715 Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({}),
3716 ),
3717 ),
3718 );
3719 yield* recovery.prepareForShutdown;
3720 yield* recovery.prepareForShutdown;
3721 const effects = yield* outbox.listByCommandId(
3722 CommandId.make(`command:restart-prepare:${runId}`),
3723 );
3724 assert.lengthOf(effects, 1);
3725 assert.equal(effects[0]?.status, "pending");
3726 assert.deepEqual(effects[0]?.request, {
3727 type: "provider-runtime.continue",
3728 sourceRunId: runId,
3729 processLossCount: 0,
3730 });
3731 yield* outbox.cancelUnsettled({
3732 threadId,
3733 effectTypes: ["provider-runtime.continue"],
3734 reason: "fixture complete",
3735 });
3736 }),
3737 );
3738
3739 it.effect("leaves restart continuation pending until normal worker claims are enabled", () =>
3740 Effect.gen(function* () {
3741 const outbox = yield* EffectOutbox.EffectOutboxV2;
3742 const threadId = ThreadId.make("thread:activation-restart");
3743 const commandId = CommandId.make("command:activation-restart");
3744 yield* outbox.enqueue([
3745 {
3746 id: "effect:activation-a-restart",
3747 commandId,
3748 threadId,
3749 request: { type: "provider-runtime.continue", sourceRunId: RunId.make("run:activation") },
3750 },
3751 {
3752 id: "effect:activation-b-cleanup",
3753 commandId,
3754 threadId,
3755 request: { type: "terminal.cleanup" },
3756 },
3757 ]);
3758 const cleanup = yield* outbox.claimNext({
3759 workerId: "recovery",
3760 excludeRestartContinuations: true,
3761 });
3762 assert.isTrue(Option.isSome(cleanup));
3763 if (Option.isSome(cleanup)) {
3764 assert.equal(cleanup.value.request.type, "terminal.cleanup");
3765 yield* outbox.succeed({ effectId: cleanup.value.id, workerId: "recovery" });
3766 }
3767 assert.isTrue(
3768 Option.isNone(
3769 yield* outbox.claimNext({
3770 workerId: "recovery",
3771 excludeRestartContinuations: true,
3772 }),
3773 ),
3774 );
3775 const pending = yield* outbox.get("effect:activation-a-restart");
3776 assert.isTrue(Option.isSome(pending));
3777 if (Option.isSome(pending)) assert.equal(pending.value.status, "pending");
3778 const resumed = yield* outbox.claimNext({ workerId: "activated" });
3779 assert.isTrue(Option.isSome(resumed));
3780 if (Option.isSome(resumed)) {
3781 assert.equal(resumed.value.request.type, "provider-runtime.continue");
3782 yield* outbox.succeed({ effectId: resumed.value.id, workerId: "activated" });
3783 }
3784 }),
3785 );
3786
3787 it.effect.each([
3788 { type: "terminal.cleanup" },
3789 { type: "provider-runtime.continue", sourceRunId: RunId.make("run:restart-replay") },
3790 ] as const)(
3791 "retires live provider effects and requeues $type after process loss",
3792 (replayRequest) =>
3793 Effect.gen(function* () {
3794 const outbox = yield* EffectOutbox.EffectOutboxV2;
3795 const commandId = CommandId.make(
3796 `command:foundation-reclaim-running:${replayRequest.type}`,
3797 );
3798 yield* outbox.enqueue([
3799 {
3800 id: `effect:a-foundation-cancel-provider-turn:${replayRequest.type}`,
3801 commandId,
3802 threadId: ThreadId.make(`thread:foundation-reclaim-running:${replayRequest.type}`),
3803 request: {
3804 type: "provider-turn.start",
3805 runId: RunId.make(`run:foundation-reclaim-running:${replayRequest.type}`),
3806 },
3807 },
3808 {
3809 id: `effect:b-foundation-requeue-cleanup:${replayRequest.type}`,
3810 commandId,
3811 threadId: ThreadId.make(`thread:foundation-reclaim-cleanup:${replayRequest.type}`),
3812 request: replayRequest,
3813 },
3814 ]);
3815 assert.isTrue(Option.isSome(yield* outbox.claimNext({ workerId: "crashed-worker" })));
3816 assert.isTrue(Option.isSome(yield* outbox.claimNext({ workerId: "crashed-worker" })));
3817 assert.deepEqual(yield* outbox.reconcileAfterProcessLoss, {
3818 cancelled: 1,
3819 requeued: 1,
3820 });
3821 const cancelled = yield* outbox.get(
3822 `effect:a-foundation-cancel-provider-turn:${replayRequest.type}`,
3823 );
3824 assert.isTrue(Option.isSome(cancelled));
3825 if (Option.isSome(cancelled)) assert.equal(cancelled.value.status, "cancelled");
3826
3827 const reclaimed = yield* outbox.claimNext({
3828 workerId: "recovery-worker",
3829 });
3830 assert.isTrue(Option.isSome(reclaimed));
3831 if (Option.isSome(reclaimed)) {
3832 assert.equal(reclaimed.value.request.type, replayRequest.type);
3833 assert.equal(reclaimed.value.attemptCount, 2);
3834 yield* outbox.succeed({ effectId: reclaimed.value.id, workerId: "recovery-worker" });
3835 }
3836 }),
3837 );
3838
3839 it.effect("atomically cancels stale runs and their process-bound effects", () =>
3840 Effect.gen(function* () {
3841 const eventSink = yield* EventSink.EventSinkV2;
3842 const outbox = yield* EffectOutbox.EffectOutboxV2;
3843 const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
3844 const now = yield* DateTime.now;
3845 const threadId = ThreadId.make("thread:foundation-process-loss");
3846 const runId = RunId.make("run:foundation-process-loss");
3847 const commandId = CommandId.make("command:foundation-process-loss");
3848 const thread = makeThread(threadId, now);
3849 yield* eventSink.commitCommand({
3850 commandId,
3851 threadId,
3852 commandType: "foundation.process-loss",
3853 acceptedAt: now,
3854 events: [
3855 threadCreatedEvent({ id: "event:foundation-process-loss:thread", thread, now }),
3856 {
3857 id: EventId.make("event:foundation-process-loss:run"),
3858 type: "run.created",
3859 threadId,
3860 runId,
3861 providerInstanceId,
3862 occurredAt: now,
3863 payload: {
3864 id: runId,
3865 threadId,
3866 ordinal: 1,
3867 providerInstanceId,
3868 modelSelection,
3869 providerThreadId: null,
3870 userMessageId: MessageId.make("message:foundation-process-loss"),
3871 rootNodeId: null,
3872 activeAttemptId: null,
3873 status: "starting",
3874 queuePosition: null,
3875 requestedAt: now,
3876 startedAt: null,
3877 completedAt: null,
3878 checkpointId: null,
3879 contextHandoffId: null,
3880 },
3881 },
3882 ],
3883 effects: [
3884 {
3885 id: "effect:foundation-process-loss",
3886 commandId,
3887 threadId,
3888 request: { type: "provider-turn.start", runId },
3889 },
3890 ],
3891 });
3892 assert.isTrue(Option.isSome(yield* outbox.claimNext({ workerId: "crashed-worker" })));
3893
3894 const recovery = yield* ProviderRuntimeRecovery.make.pipe(
3895 Effect.provide(ServerSettings.layerTest()),
3896 Effect.provideService(
3897 EffectWorker.OrchestrationEffectWorkerV2,
3898 EffectWorker.OrchestrationEffectWorkerV2.of({
3899 awaitWork: Effect.void,
3900 runRecoveryOnce: Effect.succeed(false),
3901 runOnce: () => Effect.succeed(false),
3902 nextClaimableAt: () => Effect.succeedNone,
3903 drain: () => Effect.succeed(0),
3904 }),
3905 ),
3906 );
3907 const first = yield* recovery.recover;
3908 assert.equal(first.terminalizedRuns, 1);
3909 assert.equal(first.retiredEffects, 1);
3910 const projection = yield* projectionStore.getThreadProjection(threadId);
3911 assert.equal(projection.runs[0]?.status, "cancelled");
3912 const effect = yield* outbox.get("effect:foundation-process-loss");
3913 assert.isTrue(Option.isSome(effect));
3914 if (Option.isSome(effect)) assert.equal(effect.value.status, "cancelled");
3915
3916 const second = yield* recovery.recover;
3917 assert.equal(second.terminalizedRuns, 0);
3918 assert.equal(second.retiredEffects, 0);
3919 }),
3920 );
3921
3922 it.effect("settles a native subagent's child thread when its provider process is gone", () =>
3923 Effect.gen(function* () {
3924 const eventSink = yield* EventSink.EventSinkV2;
3925 const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
3926 const now = yield* DateTime.now;
3927 const parentId = ThreadId.make("thread:foundation-native-subagent-parent");
3928 const childId = ThreadId.make("thread:foundation-native-subagent-child");
3929 const runId = RunId.make("run:foundation-native-subagent");
3930 const subagentId = NodeId.make("node:foundation-native-subagent");
3931 const childRootId = NodeId.make("node:foundation-native-subagent-child-root");
3932 const parent = makeThread(parentId, now);
3933 const child: OrchestrationV2AppThread = {
3934 ...makeThread(childId, now),
3935 createdBy: "agent",
3936 creationSource: "provider",
3937 lineage: {
3938 parentThreadId: parentId,
3939 relationshipToParent: "subagent",
3940 rootThreadId: parentId,
3941 },
3942 forkedFrom: { type: "node", nodeId: subagentId },
3943 };
3944 const node = (input: {
3945 readonly id: NodeId;
3946 readonly threadId: ThreadId;
3947 readonly runId: RunId | null;
3948 readonly kind: "root_turn" | "subagent";
3949 }) => ({
3950 ...input,
3951 parentNodeId: null,
3952 rootNodeId: input.id,
3953 status: "running" as const,
3954 countsForRun: false,
3955 providerThreadId: null,
3956 providerTurnId: null,
3957 nativeItemRef: null,
3958 runtimeRequestId: null,
3959 checkpointScopeId: null,
3960 startedAt: now,
3961 completedAt: null,
3962 });
3963 yield* eventSink.commitCommand({
3964 commandId: CommandId.make("command:foundation-native-subagent"),
3965 threadId: parentId,
3966 commandType: "foundation.native-subagent",
3967 acceptedAt: now,
3968 events: [
3969 threadCreatedEvent({
3970 id: "event:foundation-native-subagent:parent",
3971 thread: parent,
3972 now,
3973 }),
3974 threadCreatedEvent({ id: "event:foundation-native-subagent:child", thread: child, now }),
3975 {
3976 id: EventId.make("event:foundation-native-subagent:run"),
3977 type: "run.created",
3978 threadId: parentId,
3979 runId,
3980 providerInstanceId,
3981 occurredAt: now,
3982 payload: {
3983 id: runId,
3984 threadId: parentId,
3985 ordinal: 1,
3986 providerInstanceId,
3987 modelSelection,
3988 providerThreadId: null,
3989 userMessageId: MessageId.make("message:foundation-native-subagent"),
3990 rootNodeId: null,
3991 activeAttemptId: null,
3992 status: "completed",
3993 queuePosition: null,
3994 requestedAt: now,
3995 startedAt: now,
3996 completedAt: now,
3997 checkpointId: null,
3998 contextHandoffId: null,
3999 },
4000 },
4001 {
4002 id: EventId.make("event:foundation-native-subagent:subagent-node"),
4003 type: "node.updated",
4004 threadId: parentId,
4005 runId,
4006 nodeId: subagentId,
4007 occurredAt: now,
4008 payload: node({ id: subagentId, threadId: parentId, runId, kind: "subagent" }),
4009 },
4010 {
4011 id: EventId.make("event:foundation-native-subagent:child-root"),
4012 type: "node.updated",
4013 threadId: childId,
4014 nodeId: childRootId,
4015 occurredAt: now,
4016 payload: node({ id: childRootId, threadId: childId, runId: null, kind: "root_turn" }),
4017 },
4018 {
4019 id: EventId.make("event:foundation-native-subagent:child-progress"),
4020 type: "turn-item.updated",
4021 threadId: childId,
4022 nodeId: childRootId,
4023 occurredAt: now,
4024 payload: {
4025 id: TurnItemId.make("item:foundation-native-subagent:progress"),
4026 threadId: childId,
4027 runId: null,
4028 nodeId: childRootId,
4029 providerThreadId: null,
4030 providerTurnId: null,
4031 nativeItemRef: null,
4032 parentItemId: null,
4033 ordinal: 101,
4034 type: "reasoning",
4035 status: "running",
4036 title: "Thinking",
4037 startedAt: now,
4038 completedAt: null,
4039 updatedAt: now,
4040 text: "Checking the diff.",
4041 streaming: true,
4042 },
4043 },
4044 {
4045 id: EventId.make("event:foundation-native-subagent:subagent"),
4046 type: "subagent.updated",
4047 threadId: parentId,
4048 runId,
4049 nodeId: subagentId,
4050 driver: providerDriver,
4051 providerInstanceId,
4052 occurredAt: now,
4053 payload: {
4054 id: subagentId,
4055 threadId: parentId,
4056 runId,
4057 parentNodeId: subagentId,
4058 origin: "provider_native",
4059 createdBy: "agent",
4060 driver: providerDriver,
4061 providerInstanceId,
4062 providerThreadId: null,
4063 childThreadId: childId,
4064 nativeTaskRef: null,
4065 prompt: "Audit the adapters",
4066 title: null,
4067 model: null,
4068 status: "running",
4069 result: null,
4070 startedAt: now,
4071 completedAt: null,
4072 updatedAt: now,
4073 },
4074 },
4075 {
4076 id: EventId.make("event:foundation-native-subagent:item"),
4077 type: "turn-item.updated",
4078 threadId: parentId,
4079 runId,
4080 nodeId: subagentId,
4081 occurredAt: now,
4082 payload: {
4083 id: TurnItemId.make("item:foundation-native-subagent"),
4084 threadId: parentId,
4085 runId,
4086 nodeId: subagentId,
4087 providerThreadId: null,
4088 providerTurnId: null,
4089 nativeItemRef: null,
4090 parentItemId: null,
4091 ordinal: 1,
4092 type: "subagent",
4093 status: "interrupted",
4094 title: null,
4095 startedAt: now,
4096 completedAt: null,
4097 updatedAt: now,
4098 subagentId,
4099 origin: "provider_native",
4100 driver: providerDriver,
4101 providerInstanceId,
4102 childThreadId: childId,
4103 prompt: "Audit the adapters",
4104 result: null,
4105 },
4106 },
4107 ],
4108 effects: [],
4109 });
4110
4111 const recovery = yield* ProviderRuntimeRecovery.make.pipe(
4112 Effect.provide(ServerSettings.layerTest()),
4113 Effect.provideService(
4114 EffectWorker.OrchestrationEffectWorkerV2,
4115 EffectWorker.OrchestrationEffectWorkerV2.of({
4116 awaitWork: Effect.void,
4117 runRecoveryOnce: Effect.succeed(false),
4118 runOnce: () => Effect.succeed(false),
4119 nextClaimableAt: () => Effect.succeedNone,
4120 drain: () => Effect.succeed(0),
4121 }),
4122 ),
4123 );
4124 assert.include(yield* projectionStore.getRecoveryThreadIds("runtime"), childId);
4125 yield* recovery.recover;
4126
4127 const parentProjection = yield* projectionStore.getThreadProjection(parentId);
4128 assert.equal(parentProjection.subagents[0]?.status, "cancelled");
4129 assert.equal(
4130 parentProjection.nodes.find((node) => node.id === subagentId)?.status,
4131 "cancelled",
4132 );
4133 const childProjection = yield* projectionStore.getThreadProjection(childId);
4134 const childRoot = childProjection.nodes.find((candidate) => candidate.id === childRootId);
4135 assert.equal(childRoot?.status, "cancelled");
4136 assert.isNotNull(childRoot?.completedAt ?? null);
4137 const progress = childProjection.turnItems.find((item) => item.type === "reasoning");
4138 assert.equal(progress?.status, "cancelled");
4139 assert.isFalse(progress?.type === "reasoning" && progress.streaming);
4140 assert.isNotNull(progress?.completedAt ?? null);
4141 assert.notInclude(yield* projectionStore.getRecoveryThreadIds("runtime"), childId);
4142 }),
4143 );
4144
4145 it.effect("allocates collision-free positions beyond 100 items and rebuilds equivalently", () =>
4146 Effect.gen(function* () {
4147 const eventSink = yield* EventSink.EventSinkV2;
4148 const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
4149 const maintenance = yield* ProjectionReplay.ProjectionReplay;
4150 const sql = yield* SqlClient.SqlClient;
4151 const now = yield* DateTime.now;
4152 const threadId = ThreadId.make("thread:foundation-many-items");
4153 const runId = RunId.make("run:foundation-many-items");
4154 const providerThreadId = ProviderThreadId.make("provider-thread:foundation-many-items");
4155 const thread = makeThread(threadId, now);
4156 const providerThreadEvent = {
4157 id: EventId.make("event:foundation-many-items:provider-thread"),
4158 type: "provider-thread.updated" as const,
4159 threadId,
4160 providerInstanceId,
4161 occurredAt: now,
4162 payload: {
4163 id: providerThreadId,
4164 driver: providerDriver,
4165 providerInstanceId,
4166 providerSessionId: null,
4167 appThreadId: threadId,
4168 ownerNodeId: null,
4169 nativeThreadRef: null,
4170 nativeConversationHeadRef: null,
4171 status: "active" as const,
4172 firstRunOrdinal: 1,
4173 lastRunOrdinal: 1,
4174 handoffIds: [],
4175 forkedFrom: null,
4176 createdAt: now,
4177 updatedAt: now,
4178 },
4179 } satisfies OrchestrationV2DomainEvent;
4180 const runEvent = {
4181 id: EventId.make("event:foundation-many-items:run"),
4182 type: "run.created" as const,
4183 threadId,
4184 runId,
4185 providerInstanceId,
4186 occurredAt: now,
4187 payload: {
4188 id: runId,
4189 threadId,
4190 ordinal: 1,
4191 providerInstanceId,
4192 modelSelection,
4193 providerThreadId: null,
4194 userMessageId: MessageId.make("message:foundation-many-items"),
4195 rootNodeId: null,
4196 activeAttemptId: null,
4197 status: "completed" as const,
4198 queuePosition: null,
4199 requestedAt: now,
4200 startedAt: now,
4201 completedAt: now,
4202 checkpointId: null,
4203 contextHandoffId: null,
4204 },
4205 } satisfies OrchestrationV2DomainEvent;
4206 const items = Array.from({ length: 151 }, (_, index) => ({
4207 id: TurnItemId.make(`turn-item:foundation-many-items:${index}`),
4208 threadId,
4209 runId,
4210 nodeId: null,
4211 providerThreadId: null,
4212 providerTurnId: null,
4213 nativeItemRef: null,
4214 parentItemId: null,
4215 ordinal: index % 3,
4216 status: "completed" as const,
4217 title: null,
4218 startedAt: now,
4219 completedAt: now,
4220 updatedAt: now,
4221 type: "dynamic_tool" as const,
4222 toolName: `tool-${index}`,
4223 input: { index },
4224 output: { completed: true },
4225 }));
4226 const itemEvents = items.map(
4227 (item, index) =>
4228 ({
4229 id: EventId.make(`event:foundation-many-items:item:${index}`),
4230 type: "turn-item.updated",
4231 threadId,
4232 runId,
4233 providerInstanceId,
4234 occurredAt: now,
4235 payload: item,
4236 }) satisfies OrchestrationV2DomainEvent,
4237 );
4238
4239 yield* eventSink.write({
4240 events: [
4241 threadCreatedEvent({ id: "event:foundation-many-items:thread", thread, now }),
4242 providerThreadEvent,
4243 runEvent,
4244 ...itemEvents,
4245 ],
4246 });
4247 const beforeUpdate = yield* projectionStore.getThreadProjection(threadId);
4248 assert.equal(beforeUpdate.thread.activeProviderThreadId, providerThreadId);
4249 const ordinals = beforeUpdate.turnItems.map((item) => item.ordinal);
4250 assert.lengthOf(ordinals, 151);
4251 assert.equal(new Set(ordinals).size, 151);
4252 assert.isTrue(ordinals.every((ordinal) => ordinal > 1_000_000));
4253 assert.isTrue(
4254 ordinals.every((ordinal, index) => index === 0 || ordinal > ordinals[index - 1]!),
4255 );
4256
4257 yield* eventSink.write({
4258 events: [
4259 {
4260 id: EventId.make("event:foundation-many-items:update"),
4261 type: "turn-item.updated",
4262 threadId,
4263 runId,
4264 providerInstanceId,
4265 occurredAt: now,
4266 payload: { ...items[0]!, ordinal: 99_999_999, title: "Updated" },
4267 },
4268 ],
4269 });
4270 const afterUpdate = yield* projectionStore.getThreadProjection(threadId);
4271 assert.equal(afterUpdate.turnItems[0]?.ordinal, ordinals[0]);
4272 assert.equal((yield* projectionStore.getThreadShell(threadId))?.itemCount, 151);
4273 assert.equal((yield* maintenance.verify).valid, true);
4274
4275 yield* sql`
4276 UPDATE orchestration_v2_projection_turn_items
4277 SET payload_json = '{}'
4278 WHERE turn_item_id = ${items[75]!.id}
4279 `;
4280 const broken = yield* maintenance.verify;
4281 assert.isFalse(broken.valid);
4282 assert.deepEqual(broken.unreadableThreadIds, [threadId]);
4283
4284 const rebuilt = yield* maintenance.rebuild;
4285 assert.isTrue(rebuilt.valid);
4286 assert.lengthOf((yield* projectionStore.getThreadProjection(threadId)).turnItems, 151);
4287 assert.equal((yield* projectionStore.getThreadShell(threadId))?.itemCount, 151);
4288 }),
4289 );
4290 });
4291
4292 it.effect.each(["direct", "provider-batch", "cancel-provider-batch"] as const)(
4293 "publishes live events in commit order across concurrent writers with %s",
4294 (mode) =>
4295 Effect.gen(function* () {
4296 const firstCommitted = yield* Deferred.make<void>();
4297 const releaseFirst = yield* Deferred.make<void>();
4298 const layerPausingOutbox = Layer.effect(
4299 EffectOutbox.EffectOutboxV2,
4300 Effect.gen(function* () {
4301 const delegate = yield* EffectOutbox.EffectOutboxV2;
4302 return EffectOutbox.EffectOutboxV2.of({
4303 ...delegate,
4304 notifyAvailable: (count) =>
4305 Deferred.succeed(firstCommitted, undefined).pipe(
4306 Effect.andThen(Deferred.await(releaseFirst)),
4307 Effect.andThen(delegate.notifyAvailable(count)),
4308 ),
4309 });
4310 }),
4311 ).pipe(Layer.provide(layerEffectOutboxProvided));
4312 const layerEventSink = EventSink.layerFromStores.pipe(
4313 Layer.provide(
4314 Layer.mergeAll(
4315 layerStoresProvided,
4316 layerPausingOutbox,
4317 layerCommandReceiptStoreProvided,
4318 ProjectStore.layer.pipe(Layer.provide(layerDatabase)),
4319 TurnItemPositionStore.layer.pipe(Layer.provide(layerDatabase)),
4320 ThreadViewers.layer,
4321 ),
4322 ),
4323 );
4324
4325 yield* Effect.gen(function* () {
4326 const eventSink = yield* EventSink.EventSinkV2;
4327 const now = yield* DateTime.now;
4328 const first = makeThread(ThreadId.make("thread:foundation-publish-order:first"), now);
4329 const second = makeThread(ThreadId.make("thread:foundation-publish-order:second"), now);
4330 const published = yield* eventSink
4331 .stream({ afterSequence: yield* eventSink.latestSequence() })
4332 .pipe(Stream.take(2), Stream.runCollect, Effect.forkScoped({ startImmediately: true }));
4333
4334 const firstWrite = yield* eventSink
4335 .writeWithEffects({
4336 events: [
4337 threadCreatedEvent({
4338 id: "event:foundation-publish-order:first",
4339 thread: first,
4340 now,
4341 }),
4342 ],
4343 effects: [
4344 {
4345 id: "effect:foundation-publish-order:first",
4346 commandId: CommandId.make("command:foundation-publish-order:first"),
4347 threadId: first.id,
4348 request: { type: "terminal.cleanup" },
4349 },
4350 ],
4351 })
4352 .pipe(
4353 (write) => (mode === "direct" ? write : eventSink.withProviderBatch(write)),
4354 Effect.forkScoped,
4355 );
4356 yield* Deferred.await(firstCommitted);
4357 const secondWrite = yield* eventSink
4358 .write({
4359 events: [
4360 threadCreatedEvent({
4361 id: "event:foundation-publish-order:second",
4362 thread: second,
4363 now,
4364 }),
4365 ],
4366 })
4367 .pipe(
4368 Effect.provideService(Scheduler.MaxOpsBeforeYield, Number.POSITIVE_INFINITY),
4369 Effect.forkScoped,
4370 );
4371 yield* Effect.yieldNow;
4372 const interrupted =
4373 mode === "cancel-provider-batch"
4374 ? yield* Fiber.interrupt(firstWrite).pipe(Effect.forkScoped({ startImmediately: true }))
4375 : undefined;
4376 yield* Deferred.succeed(releaseFirst, undefined);
4377 if (interrupted !== undefined) yield* Fiber.join(interrupted);
4378 else yield* Fiber.join(firstWrite);
4379 yield* Fiber.join(secondWrite);
4380
4381 const sequences = Array.from(yield* Fiber.join(published), (stored) => stored.sequence);
4382 assert.deepEqual(
4383 sequences,
4384 [...sequences].sort((left, right) => left - right),
4385 );
4386 }).pipe(Effect.provide(layerEventSink));
4387 }).pipe(Effect.provide(layerDatabase)),
4388 );
4389
4390 it.effect.each(["events", "receipts"] as const)(
4391 "allows writes while compaction waits for the reader to discover obsolete %s",
4392 (kind) =>
4393 Effect.gen(function* () {
4394 const fs = yield* FileSystem.FileSystem;
4395 const path = yield* Path.Path;
4396 const directory = yield* fs.makeTempDirectoryScoped({ prefix: "compaction-reader-" });
4397 const database = SqlitePersistence.layerFromPath(path.join(directory, "statev2.sqlite"));
4398 const maintenanceLayer = ProjectionMaintenance.layer.pipe(Layer.provideMerge(database));
4399 yield* Effect.gen(function* () {
4400 const sql = yield* SqlClient.SqlClient;
4401 const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2;
4402 const now = DateTime.formatIso(yield* DateTime.now);
4403 yield* sql`
4404 INSERT INTO orchestration_v2_legacy_imports (
4405 thread_id, source_updated_at, shell_imported_at, transcript_imported_at,
4406 imported_message_count, last_error
4407 ) VALUES ('imported', ${now}, ${now}, ${now}, 1, NULL)
4408 `;
4409 for (const threadId of ["imported", "pending"]) {
4410 if (kind === "events") {
4411 yield* sql`
4412 INSERT INTO orchestration_events (
4413 event_id, aggregate_kind, stream_id, stream_version, event_type,
4414 occurred_at, actor_kind, payload_json, metadata_json, application_event_version
4415 ) VALUES (
4416 ${threadId}, 'thread', ${threadId}, 1, 'thread.message-appended',
4417 ${now}, 'user', '{}', '{}', 1
4418 )
4419 `;
4420 } else {
4421 yield* sql`
4422 INSERT INTO orchestration_command_receipts (
4423 command_id, aggregate_kind, aggregate_id, accepted_at, result_sequence,
4424 status, command_type
4425 ) VALUES (${threadId}, 'thread', ${threadId}, ${now}, 1, 'accepted', 'legacy')
4426 `;
4427 }
4428 }
4429 yield* sql`CREATE TABLE concurrent_writes (value TEXT)`;
4430 const readerHeld = yield* Deferred.make<void>();
4431 const releaseReader = yield* Deferred.make<void>();
4432 const discoveryStarted = yield* Deferred.make<Tracer.Span>();
4433 const observeDiscovery: Statement.Transformer = (statement, _sql, _fiber, span) => {
4434 const [query] = statement.compile();
4435 const pageQuery =
4436 kind === "events" ? /^\s*SELECT\s+event\.sequence/ : /^\s*SELECT\s+receipt\.rowid/;
4437 return pageQuery.test(query)
4438 ? Deferred.succeed(discoveryStarted, span).pipe(
4439 Effect.andThen(Deferred.await(readerHeld)),
4440 Effect.as(statement),
4441 )
4442 : Effect.succeed(statement);
4443 };
4444 const compacted = yield* Deferred.make<void>();
4445 const compaction = yield* maintenance.pruneEventStore().pipe(
4446 Effect.provideService(Statement.CurrentTransformer, observeDiscovery),
4447 Effect.provideService(Statement.SpanPropagationEnabled, true),
4448 Effect.tap(() => Deferred.succeed(compacted, undefined)),
4449 Effect.forkChild,
4450 );
4451 const discoverySpan = yield* Deferred.await(discoveryStarted);
4452 // Let the initial bounds read finish before holding the reader for this page.
4453 const reader = yield* sql
4454 .withTransaction(
4455 Deferred.succeed(readerHeld, undefined).pipe(
4456 Effect.andThen(Deferred.await(releaseReader)),
4457 ),
4458 )
4459 .pipe(NodeSqliteClient.readOnly, Effect.forkChild);
4460 yield* Deferred.await(readerHeld);
4461 yield* sql`INSERT INTO concurrent_writes VALUES ('committed')`;
4462 assert.isFalse(yield* Deferred.isDone(compacted));
4463 assert.deepEqual(yield* sql`SELECT value FROM concurrent_writes`, [{ value: "committed" }]);
4464 if (kind === "events") {
4465 yield* sql`
4466 INSERT INTO orchestration_events (
4467 event_id, aggregate_kind, stream_id, stream_version, event_type,
4468 occurred_at, actor_kind, payload_json, metadata_json, application_event_version
4469 ) VALUES (
4470 'after-bound', 'thread', 'imported', 2, 'thread.message-appended',
4471 ${now}, 'user', '{}', '{}', 1
4472 )
4473 `;
4474 } else {
4475 yield* sql`
4476 INSERT INTO orchestration_command_receipts (
4477 command_id, aggregate_kind, aggregate_id, accepted_at, result_sequence,
4478 status, command_type
4479 ) VALUES ('after-bound', 'thread', 'imported', ${now}, 1, 'accepted', 'legacy')
4480 `;
4481 }
4482 yield* Deferred.succeed(releaseReader, undefined);
4483 yield* Fiber.join(reader);
4484 const result = yield* Fiber.join(compaction);
4485 assert.equal(discoverySpan.attributes.get("db.connection"), "reader");
4486 assert.equal(result.deletedEventCount, kind === "events" ? 1 : 0);
4487 assert.equal(result.deletedReceiptCount, kind === "receipts" ? 1 : 0);
4488 const retained =
4489 kind === "events"
4490 ? yield* sql`SELECT event_id AS id FROM orchestration_events ORDER BY sequence`
4491 : yield* sql`SELECT command_id AS id FROM orchestration_command_receipts ORDER BY rowid`;
4492 assert.deepEqual(retained, [{ id: "pending" }, { id: "after-bound" }]);
4493 }).pipe(Effect.provide(maintenanceLayer));
4494 }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
4495 );
4496
4497 it.effect("routes projection node lookups to the reader connection", () =>
4498 Effect.gen(function* () {
4499 const fs = yield* FileSystem.FileSystem;
4500 const path = yield* Path.Path;
4501 const directory = yield* fs.makeTempDirectoryScoped({ prefix: "projection-reader-" });
4502 const database = SqlitePersistence.layerFromPath(path.join(directory, "statev2.sqlite"));
4503 const projectionLayer = ProjectionStore.layer.pipe(Layer.provideMerge(database));
4504 yield* Effect.gen(function* () {
4505 const projections = yield* ProjectionStore.ProjectionStoreV2;
4506 const now = yield* DateTime.now;
4507 const nodeId = NodeId.make("node:reader-routing");
4508 const node = {
4509 id: nodeId,
4510 threadId: ThreadId.make("thread:reader-routing"),
4511 runId: null,
4512 parentNodeId: null,
4513 rootNodeId: nodeId,
4514 kind: "subagent" as const,
4515 status: "running" as const,
4516 countsForRun: false,
4517 providerThreadId: null,
4518 providerTurnId: null,
4519 nativeItemRef: null,
4520 runtimeRequestId: null,
4521 checkpointScopeId: null,
4522 startedAt: now,
4523 completedAt: null,
4524 };
4525 const spans: Array<Tracer.Span> = [];
4526 const observe: Statement.Transformer = (statement, _sql, _fiber, span) => {
4527 const [query] = statement.compile();
4528 if (/orchestration_v2_projection_nodes WHERE node_id/.test(query)) {
4529 spans.push(span);
4530 }
4531 return Effect.succeed(statement);
4532 };
4533 yield* Effect.gen(function* () {
4534 assert.isFalse(yield* projections.isNodeStored(node));
4535 }).pipe(
4536 Effect.provideService(Statement.CurrentTransformer, observe),
4537 Effect.provideService(Statement.SpanPropagationEnabled, true),
4538 );
4539 assert.equal(spans.length, 1);
4540 assert.isTrue(
4541 spans.every((span) => span.attributes.get("db.connection") === "reader"),
4542 JSON.stringify(spans.map((span) => span.attributes.get("db.connection"))),
4543 );
4544 }).pipe(Effect.provide(projectionLayer));
4545 }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
4546 );
4547
4548 it.effect("loads a shell snapshot while the writer connection is occupied", () =>
4549 Effect.gen(function* () {
4550 const fs = yield* FileSystem.FileSystem;
4551 const path = yield* Path.Path;
4552 const directory = yield* fs.makeTempDirectoryScoped({ prefix: "shell-reader-" });
4553 const database = SqlitePersistence.layerFromPath(path.join(directory, "statev2.sqlite"));
4554 const projectionLayer = ProjectionStore.layer.pipe(Layer.provideMerge(database));
4555 yield* Effect.gen(function* () {
4556 const sql = yield* SqlClient.SqlClient;
4557 const projections = yield* ProjectionStore.ProjectionStoreV2;
4558 const held = yield* Deferred.make<void>();
4559 const release = yield* Deferred.make<void>();
4560 const writer = yield* sql
4561 .withTransaction(
4562 sql`SELECT 1`.pipe(
4563 Effect.andThen(Deferred.succeed(held, undefined)),
4564 Effect.andThen(Deferred.await(release)),
4565 ),
4566 )
4567 .pipe(Effect.forkChild);
4568 yield* Deferred.await(held);
4569 const snapshot = yield* projections.getShellSnapshot();
4570 yield* Deferred.succeed(release, undefined);
4571 yield* Fiber.join(writer);
4572 assert.deepEqual(snapshot.threads, []);
4573 }).pipe(Effect.provide(projectionLayer));
4574 }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)),
4575 );
4576
4577 it.effect.each(["cancel transaction", "fail transaction", "cancel publication"] as const)(
4578 "preserves event consistency and allows later writes after %s",
4579 (scenario) =>
4580 Effect.gen(function* () {
4581 const paused = yield* Deferred.make<void>();
4582 const release = yield* Deferred.make<void>();
4583 const committed = scenario === "cancel publication";
4584 const pause = Deferred.succeed(paused, undefined).pipe(
4585 Effect.andThen(Deferred.await(release)),
4586 );
4587 const layerPausingOutbox = Layer.effect(
4588 EffectOutbox.EffectOutboxV2,
4589 Effect.gen(function* () {
4590 const delegate = yield* EffectOutbox.EffectOutboxV2;
4591 return EffectOutbox.EffectOutboxV2.of({
4592 ...delegate,
4593 enqueue: (effects) =>
4594 delegate
4595 .enqueue(effects)
4596 .pipe(
4597 Effect.andThen(!committed && effects.length > 0 ? pause : Effect.void),
4598 Effect.andThen(
4599 scenario === "fail transaction" && effects.length > 0
4600 ? Effect.die("Injected transaction failure")
4601 : Effect.void,
4602 ),
4603 ),
4604 notifyAvailable: (count) =>
4605 (committed ? pause : Effect.void).pipe(
4606 Effect.andThen(delegate.notifyAvailable(count)),
4607 ),
4608 });
4609 }),
4610 ).pipe(Layer.provide(layerEffectOutboxProvided));
4611 const layerEventSink = EventSink.layerFromStores.pipe(
4612 Layer.provide(
4613 Layer.mergeAll(
4614 layerStoresProvided,
4615 layerPausingOutbox,
4616 layerCommandReceiptStoreProvided,
4617 ProjectStore.layer.pipe(Layer.provide(layerDatabase)),
4618 TurnItemPositionStore.layer.pipe(Layer.provide(layerDatabase)),
4619 ThreadViewers.layer,
4620 ),
4621 ),
4622 );
4623
4624 yield* Effect.gen(function* () {
4625 const eventSink = yield* EventSink.EventSinkV2;
4626 const projections = yield* ProjectionStore.ProjectionStoreV2;
4627 const outbox = yield* EffectOutbox.EffectOutboxV2;
4628 const now = yield* DateTime.now;
4629 const first = makeThread(ThreadId.make("thread:interrupted-publish:first"), now);
4630 const marker = makeThread(ThreadId.make("thread:interrupted-publish:marker"), now);
4631 const published = yield* eventSink
4632 .stream({ afterSequence: yield* eventSink.latestSequence() })
4633 .pipe(
4634 Stream.takeUntil((stored) => stored.event.threadId === marker.id),
4635 Stream.runCollect,
4636 Effect.forkScoped({ startImmediately: true }),
4637 );
4638 const writer = yield* eventSink
4639 .writeWithEffects({
4640 events: [
4641 threadCreatedEvent({ id: "event:interrupted-publish:first", thread: first, now }),
4642 ],
4643 effects: [
4644 {
4645 id: "effect:interrupted-publish:first",
4646 commandId: CommandId.make("command:interrupted-publish:first"),
4647 threadId: first.id,
4648 request: { type: "terminal.cleanup" },
4649 },
4650 ],
4651 })
4652 .pipe(Effect.forkScoped);
4653 yield* Deferred.await(paused);
4654 if (scenario !== "fail transaction") writer.interruptUnsafe();
4655 if (scenario !== "cancel transaction") yield* Deferred.succeed(release, undefined);
4656 assert.equal((yield* Fiber.await(writer))._tag, "Failure");
4657 assert.equal(
4658 (yield* projections.getThreadShell(first.id))?.id ?? null,
4659 committed ? first.id : null,
4660 );
4661 assert.equal(
4662 Option.isSome(yield* outbox.get("effect:interrupted-publish:first")),
4663 committed,
4664 );
4665 yield* eventSink.write({
4666 events: [
4667 threadCreatedEvent({ id: "event:interrupted-publish:marker", thread: marker, now }),
4668 ],
4669 });
4670 assert.deepEqual(
4671 Array.from(yield* Fiber.join(published), (stored) => stored.event.threadId),
4672 committed ? [first.id, marker.id] : [marker.id],
4673 );
4674 }).pipe(
4675 Effect.provide(
4676 Layer.mergeAll(layerEventSink, layerStoresProvided, layerEffectOutboxProvided),
4677 ),
4678 );
4679 }).pipe(Effect.provide(layerDatabase)),
4680 );