The best way to control Claude Code, Codex and more from any computer or phone
perf(server): batch queued provider writes without waiting (#31656)
Yash Singh committed
Nov 5, 2026 at 22:54 UTC
bb29ca114e81230b0cb3f762c6ca2cea89bb5b89
11 files changed
+1641
-435
apps/server/src/orchestration-v2/EventSink.ts
+119
-1
@@ -12,6 +12,8 @@ import {
12
type ProjectId,
13
ThreadId,
14
} from "@t3tools/contracts";
15
+import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
16
+import * as ProviderCommitBatch from "./ProviderCommitBatch.ts";
17
import * as Context from "effect/Context";
18
import * as DateTime from "effect/DateTime";
19
import * as Effect from "effect/Effect";
@@ -22,6 +24,7 @@ import * as Semaphore from "effect/Semaphore";
24
import * as Schema from "effect/Schema";
25
import * as Stream from "effect/Stream";
26
import * as SqlClient from "effect/sql/SqlClient";
27
+import * as SqlError from "effect/sql/SqlError";
28
29
import { replayAndBufferProjectedLiveEvents } from "./LiveStreamBudget.ts";
30
import type { UnsequencedProjectEvent } from "../persistence/OrchestrationEventStore.ts";
@@ -66,7 +69,32 @@ export class EventSinkStreamError extends Schema.TaggedError<EventSinkStreamErro
69
export const EventSinkV2Error = Schema.Union([EventSinkWriteError, EventSinkStreamError]);
70
export type EventSinkV2Error = typeof EventSinkV2Error.Type;
71
72
+interface ProviderBatchPublication {
73
+ readonly publish: Array<Effect.Effect<void>>;
74
+ eventCount: number;
75
+}
76
+const CurrentProviderBatch = Context.Reference<ProviderBatchPublication | undefined>(
77
+ "t3/orchestration-v2/EventSink/CurrentProviderBatch",
78
+ { defaultValue: () => undefined },
79
+);
80
+
81
export interface EventSinkV2Shape {
82
+ readonly consumeProviderEvents: <E, R, E2, R2>(
83
+ source: Stream.Stream<ProviderAdapter.ProviderAdapterV2Event, E, R>,
84
+ process: (event: ProviderAdapter.ProviderAdapterV2Event) => Effect.Effect<void, E2, R2>,
85
+ after: Effect.Effect<boolean, E2, R2>,
86
+ canBatch: Effect.Effect<boolean, never, R2>,
87
+ ) => Effect.Effect<
88
+ void,
89
+ E | E2 | ProviderCommitBatch.CommitBatchError | Schema.SchemaError,
90
+ R | R2
91
+ >;
92
+
93
+ /** Normalize and persist an ordered group against one transaction; publish after its commit. */
94
+ readonly withProviderBatch: <A, E, R>(
95
+ effect: Effect.Effect<A, E, R>,
96
+ ) => Effect.Effect<A, E | EventSinkV2Error, R>;
97
+
98
readonly write: (input: {
99
/** Return no events if another thread owns this native session at commit time. */
100
readonly unlessNativeThreadOwned?: Pick<
@@ -256,7 +284,7 @@ const layerBase: Layer.Layer<
284
285
// Serialize commit and publish; reordered publication makes clients drop earlier sequences.
286
const publishLane = yield* Semaphore.make(1);
259
- const commitThenPublish = <A, E, R>(
287
+ const commitThenPublishNow = <A, E, R>(
288
transaction: Effect.Effect<A, E, R>,
289
publish: (committed: A) => Effect.Effect<void>,
290
) =>
@@ -416,6 +444,8 @@ const layerBase: Layer.Layer<
444
"orchestration_v2.thread_id": input.events[0]?.threadId ?? null,
445
});
446
447
+ const batch = yield* CurrentProviderBatch;
448
+ if (batch !== undefined) batch.eventCount += input.events.length;
449
const { committed } = yield* commitThenPublish(
450
Effect.gen(function* () {
451
// Check ownership in the same transaction that claims the native session.
@@ -877,7 +907,95 @@ const layerBase: Layer.Layer<
907
);
908
};
909
910
+ const commitThenPublish = <A, E, R>(
911
+ transaction: Effect.Effect<A, E, R>,
912
+ publish: (committed: A) => Effect.Effect<void>,
913
+ ) =>
914
+ Effect.flatMap(CurrentProviderBatch, (batch) =>
915
+ batch === undefined
916
+ ? Effect.flatMap(Effect.serviceOption(sql.transactionService), (existing) =>
917
+ Option.isSome(existing)
918
+ ? commitThenPublishNow(transaction, publish)
919
+ : coordinator.barrier(commitThenPublishNow(transaction, publish)),
920
+ )
921
+ : transaction.pipe(
922
+ Effect.tap((committed) =>
923
+ Effect.sync(() => {
924
+ batch.publish.push(publish(committed));
925
+ }),
926
+ ),
927
+ ),
928
+ );
929
+
930
+ const withProviderBatch: EventSinkV2Shape["withProviderBatch"] = Effect.fn(
931
+ "orchestrationV2.EventSink.providerBatch",
932
+ )(function* <A, E, R>(effect: Effect.Effect<A, E, R>) {
933
+ if ((yield* CurrentProviderBatch) !== undefined) return yield* effect;
934
+ const batch: ProviderBatchPublication = { publish: [], eventCount: 0 };
935
+ let started = false;
936
+ const result = yield* commitThenPublishNow(
937
+ Effect.sync(() => {
938
+ started = true;
939
+ }).pipe(Effect.andThen(Effect.provideService(effect, CurrentProviderBatch, batch))),
940
+ () => Effect.forEach(batch.publish, (publish) => publish, { discard: true }),
941
+ ).pipe(
942
+ // Normalization mutates run-local state, so only retry before its transaction body starts.
943
+ Effect.retry({
944
+ while: (cause) =>
945
+ !started && SqlError.isSqlError(cause) && cause.reason._tag === "LockTimeoutError",
946
+ times: 2,
947
+ }),
948
+ Effect.mapError((cause) =>
949
+ SqlError.isSqlError(cause)
950
+ ? new EventSinkWriteError({ eventCount: batch.eventCount, cause })
951
+ : cause,
952
+ ),
953
+ );
954
+ yield* Effect.annotateCurrentSpan({
955
+ "orchestration_v2.event_count": batch.eventCount,
956
+ "orchestration_v2.batch_writes": batch.publish.length,
957
+ });
958
+ return result;
959
+ });
960
+
961
+ const coordinator = yield* ProviderCommitBatch.make(
962
+ (work) =>
963
+ withProviderBatch(work).pipe(
964
+ Effect.mapError((cause) =>
965
+ ProviderCommitBatch.isCommitBatchError(cause)
966
+ ? cause
967
+ : new ProviderCommitBatch.CommitBatchError({ cause }),
968
+ ),
969
+ ),
970
+ yield* ProviderCommitBatch.enabled,
971
+ // Each snapshot owns a savepoint and its deferred publications within the shared commit.
972
+ Effect.fnUntraced(function* (
973
+ work: Effect.Effect<void, ProviderCommitBatch.CommitBatchError>,
974
+ ) {
975
+ const batch = yield* CurrentProviderBatch;
976
+ const publicationCount = batch?.publish.length ?? 0;
977
+ const eventCount = batch?.eventCount ?? 0;
978
+ return yield* sql.withTransaction(work).pipe(
979
+ Effect.onError(() =>
980
+ Effect.sync(() => {
981
+ if (batch !== undefined) {
982
+ batch.publish.length = publicationCount;
983
+ batch.eventCount = eventCount;
984
+ }
985
+ }),
986
+ ),
987
+ Effect.mapError((cause) =>
988
+ ProviderCommitBatch.isCommitBatchError(cause)
989
+ ? cause
990
+ : new ProviderCommitBatch.CommitBatchError({ cause }),
991
+ ),
992
+ );
993
+ }),
994
+ );
995
+
996
return EventSinkV2.of({
997
+ consumeProviderEvents: coordinator.consume,
998
+ withProviderBatch,
999
write: (input) =>
1000
writeEffect({ ...input, effects: [] }).pipe(
1001
Effect.mapError(
apps/server/src/orchestration-v2/FoundationPersistence.test.ts
+330
-78
@@ -27,9 +27,14 @@ import {
27
TurnItemId,
28
} from "@t3tools/contracts";
29
import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient";
30
+import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
31
+import * as ConfigProvider from "effect/ConfigProvider";
32
+import * as Queue from "effect/Queue";
33
import * as Arr from "effect/Array";
34
+import * as Cause from "effect/Cause";
35
import * as Clock from "effect/Clock";
36
import * as DateTime from "effect/DateTime";
37
+import * as Exit from "effect/Exit";
38
import * as Deferred from "effect/Deferred";
39
import * as Duration from "effect/Duration";
40
import * as Effect from "effect/Effect";
@@ -325,6 +330,239 @@ it.effect("paces scheduled compaction while preserving retention state and fixed
330
}).pipe(Effect.scoped, Effect.provide(layerTest)),
331
);
332
333
+it.effect(
334
+ "commits an ordered provider batch atomically and publishes only after the outer commit",
335
+ () =>
336
+ Effect.gen(function* () {
337
+ const sink = yield* EventSink.EventSinkV2;
338
+ const projections = yield* ProjectionStore.ProjectionStoreV2;
339
+ const store = yield* EventStore.EventStoreV2;
340
+ const now = yield* DateTime.now;
341
+ const thread = makeThread(ThreadId.make("thread:single-stream-batch"), now);
342
+ yield* sink.write({ events: [threadCreatedEvent({ id: "batch-created", thread, now })] });
343
+ const message = (id: string): OrchestrationV2DomainEvent => ({
344
+ id: EventId.make(id),
345
+ type: "message.updated",
346
+ threadId: thread.id,
347
+ occurredAt: now,
348
+ payload: {
349
+ id: MessageId.make("batch-message"),
350
+ threadId: thread.id,
351
+ runId: null,
352
+ nodeId: null,
353
+ createdBy: "agent",
354
+ creationSource: "provider",
355
+ role: "assistant",
356
+ text: id,
357
+ streaming: true,
358
+ attachments: [],
359
+ createdAt: now,
360
+ updatedAt: now,
361
+ },
362
+ });
363
+ const highWater = yield* sink.latestSequence();
364
+ const callerError = { _tag: "CallerError" } as const;
365
+ assert.strictEqual(
366
+ yield* sink.withProviderBatch(Effect.fail(callerError)).pipe(Effect.flip),
367
+ callerError,
368
+ );
369
+ const failed = yield* sink
370
+ .withProviderBatch(
371
+ Effect.gen(function* () {
372
+ yield* sink.write({ events: [message("batch-duplicate")] });
373
+ yield* sink.write({ events: [message("batch-duplicate")] });
374
+ }),
375
+ )
376
+ .pipe(Effect.exit);
377
+ assert.isTrue(Exit.isFailure(failed));
378
+ assert.equal(yield* sink.latestSequence(), highWater);
379
+ assert.isEmpty((yield* projections.getThreadRecords(thread.id, ["messages"])).messages);
380
+
381
+ const firstPublished = yield* Deferred.make<void>();
382
+ const writesApplied = yield* Deferred.make<void>();
383
+ const release = yield* Deferred.make<void>();
384
+ const live = yield* sink.stream({ afterSequence: highWater }).pipe(
385
+ Stream.tap(() => Deferred.succeed(firstPublished, undefined)),
386
+ Stream.take(2),
387
+ Stream.runCollect,
388
+ Effect.forkScoped({ startImmediately: true }),
389
+ );
390
+ const batch = yield* sink
391
+ .withProviderBatch(
392
+ Effect.gen(function* () {
393
+ yield* sink.write({ events: [message("batch-first")] });
394
+ yield* sink.write({ events: [message("batch-second")] });
395
+ assert.equal(
396
+ (yield* projections.getThreadRecords(thread.id, ["messages"])).messages[0]?.text,
397
+ "batch-second",
398
+ );
399
+ yield* Deferred.succeed(writesApplied, undefined);
400
+ yield* Deferred.await(release);
401
+ }),
402
+ )
403
+ .pipe(Effect.forkScoped({ startImmediately: true }));
404
+ yield* Deferred.await(writesApplied);
405
+ yield* TestClock.adjust(0);
406
+ assert.isFalse(yield* Deferred.isDone(firstPublished));
407
+ assert.isUndefined(yield* Effect.sync(() => batch.pollUnsafe()));
408
+ yield* Deferred.succeed(release, undefined);
409
+ yield* Fiber.join(batch);
410
+ const published = yield* Fiber.join(live);
411
+ assert.deepEqual(
412
+ published.map((stored) => stored.event.id),
413
+ ["batch-first", "batch-second"],
414
+ );
415
+ assert.deepEqual(
416
+ yield* store.read({ afterSequence: highWater }).pipe(Stream.runCollect),
417
+ published,
418
+ );
419
+ }).pipe(Effect.provide(Layer.fresh(layerTest)), Effect.scoped),
420
+);
421
+
422
+it.effect.each(
423
+ (["none", "duplicate", "after-write"] as const).flatMap((failure) =>
424
+ [1, 10].map((streams) => ({ failure, streams })),
425
+ ),
426
+)(
427
+ "shares the writer and isolates failure=$failure across $streams independent consumers",
428
+ ({ failure, streams }) => {
429
+ const rollback = failure !== "none";
430
+ const failedIndex = streams - 1;
431
+ const jobError = { _tag: "InvalidUpdate" } as const;
432
+ const writes: number[] = [];
433
+ const tracer = Tracer.make({
434
+ span(options) {
435
+ const span = new Tracer.NativeSpan(options);
436
+ const end = span.end.bind(span);
437
+ span.end = (time, exit) => {
438
+ end(time, exit);
439
+ if (span.name === "orchestrationV2.EventSink.providerBatch" && Exit.isSuccess(exit)) {
440
+ writes.push(Number(span.attributes.get("orchestration_v2.batch_writes")));
441
+ }
442
+ };
443
+ return span;
444
+ },
445
+ });
446
+ return Effect.gen(function* () {
447
+ const sink = yield* EventSink.EventSinkV2;
448
+ const projections = yield* ProjectionStore.ProjectionStoreV2;
449
+ const now = yield* DateTime.now;
450
+ const threads = Array.from({ length: streams }, (_, index) =>
451
+ makeThread(ThreadId.make(`shared:${index}`), now),
452
+ );
453
+ yield* sink.write({
454
+ events: threads.map((thread, index) =>
455
+ threadCreatedEvent({ id: `created:${index}`, thread, now }),
456
+ ),
457
+ });
458
+ const highWater = yield* sink.latestSequence();
459
+ const store = yield* EventStore.EventStoreV2;
460
+ const publications = yield* Queue.unbounded<EventId>();
461
+ yield* sink.stream({ afterSequence: highWater }).pipe(
462
+ Stream.runForEach((stored) => Queue.offer(publications, stored.event.id)),
463
+ Effect.forkScoped({ startImmediately: true }),
464
+ );
465
+ const pulled = yield* Deferred.make<void>();
466
+ let arrivals = 0;
467
+ const inputs = yield* Effect.forEach(threads, () =>
468
+ Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event>(),
469
+ );
470
+ const consumers = yield* Effect.forEach(inputs, (input, index) =>
471
+ sink
472
+ .consumeProviderEvents(
473
+ Stream.fromQueue(input).pipe(
474
+ Stream.tap(() =>
475
+ ++arrivals === streams ? Deferred.succeed(pulled, undefined) : Effect.void,
476
+ ),
477
+ ),
478
+ (raw) =>
479
+ raw.type === "message.updated"
480
+ ? sink
481
+ .write({
482
+ events: [
483
+ {
484
+ id: EventId.make(
485
+ failure === "duplicate" && index === failedIndex
486
+ ? "created:0"
487
+ : `shared-event:${index}`,
488
+ ),
489
+ threadId: raw.message.threadId,
490
+ occurredAt: now,
491
+ type: "message.updated",
492
+ payload: raw.message,
493
+ },
494
+ ],
495
+ })
496
+ .pipe(
497
+ Effect.andThen(
498
+ failure === "after-write" && index === failedIndex
499
+ ? Effect.fail(jobError)
500
+ : Effect.void,
501
+ ),
502
+ )
503
+ : Effect.void,
504
+ Effect.succeed(false),
505
+ Effect.succeed(true),
506
+ )
507
+ .pipe(Effect.exit, Effect.forkScoped({ startImmediately: true })),
508
+ );
509
+ for (let index = 0; index < threads.length; index++) {
510
+ yield* Queue.offer(inputs[index]!, {
511
+ type: "message.updated",
512
+ driver: ProviderDriverKind.make("codex"),
513
+ message: {
514
+ id: MessageId.make(`message:${index}`),
515
+ threadId: threads[index]!.id,
516
+ runId: null,
517
+ nodeId: null,
518
+ createdBy: "agent",
519
+ creationSource: "provider",
520
+ role: "assistant",
521
+ text: `stream:${index}`,
522
+ streaming: true,
523
+ attachments: [],
524
+ createdAt: now,
525
+ updatedAt: now,
526
+ },
527
+ });
528
+ }
529
+ yield* Deferred.await(pulled);
530
+ yield* TestClock.adjust(0);
531
+ yield* sink.write({ events: [] });
532
+ if (rollback) {
533
+ const exit = yield* Fiber.join(consumers[failedIndex]!);
534
+ assert.isTrue(Exit.isFailure(exit));
535
+ if (failure === "after-write" && Exit.isFailure(exit))
536
+ assert.strictEqual(Cause.squash(exit.cause), jobError);
537
+ }
538
+ yield* TestClock.adjust(0);
539
+ const replay = yield* store.read({ afterSequence: highWater }).pipe(Stream.runCollect);
540
+ assert.deepEqual(
541
+ replay.length === 0 ? [] : yield* Queue.takeN(publications, replay.length),
542
+ replay.map((stored) => stored.event.id),
543
+ );
544
+ assert.equal(yield* Queue.size(publications), 0);
545
+ assert.deepEqual(writes, rollback && streams === 1 ? [] : [streams - (rollback ? 1 : 0)]);
546
+ assert.equal(yield* sink.latestSequence(), highWater + streams - (rollback ? 1 : 0));
547
+ for (let index = 0; index < threads.length; index++) {
548
+ assert.lengthOf(
549
+ (yield* projections.getThreadRecords(threads[index]!.id, ["messages"])).messages,
550
+ rollback && index === failedIndex ? 0 : 1,
551
+ );
552
+ }
553
+ }).pipe(
554
+ Effect.scoped,
555
+ Effect.provide(Layer.fresh(layerTest)),
556
+ Effect.withTracer(tracer),
557
+ Effect.withTracerEnabled(true),
558
+ Effect.provideService(
559
+ ConfigProvider.ConfigProvider,
560
+ ConfigProvider.fromUnknown({ T3CODE_PROVIDER_COMMIT_BATCHING: true }),
561
+ ),
562
+ );
563
+ },
564
+);
565
+
566
it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
567
it.effect("projects oversized tool bodies once before replay and live RPC retention", () =>
568
Effect.scoped(
@@ -3288,87 +3526,101 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
3526
);
3527
});
3528
3291
-it.effect("publishes live events in commit order across concurrent writers", () =>
3292
- Effect.gen(function* () {
3293
- const firstCommitted = yield* Deferred.make<void>();
3294
- const releaseFirst = yield* Deferred.make<void>();
3295
- const layerPausingOutbox = Layer.effect(
3296
- EffectOutbox.EffectOutboxV2,
3297
- Effect.gen(function* () {
3298
- const delegate = yield* EffectOutbox.EffectOutboxV2;
3299
- return EffectOutbox.EffectOutboxV2.of({
3300
- ...delegate,
3301
- notifyAvailable: (count) =>
3302
- Deferred.succeed(firstCommitted, undefined).pipe(
3303
- Effect.andThen(Deferred.await(releaseFirst)),
3304
- Effect.andThen(delegate.notifyAvailable(count)),
3305
- ),
3306
- });
3307
- }),
3308
- ).pipe(Layer.provide(layerEffectOutboxProvided));
3309
- const layerEventSink = EventSink.layerFromStores.pipe(
3310
- Layer.provide(
3311
- Layer.mergeAll(
3312
- layerStoresProvided,
3313
- layerPausingOutbox,
3314
- layerCommandReceiptStoreProvided,
3315
- ProjectStore.layer.pipe(Layer.provide(layerDatabase)),
3316
- TurnItemPositionStore.layer.pipe(Layer.provide(layerDatabase)),
3529
+it.effect.each(["direct", "provider-batch", "cancel-provider-batch"] as const)(
3530
+ "publishes live events in commit order across concurrent writers with %s",
3531
+ (mode) =>
3532
+ Effect.gen(function* () {
3533
+ const firstCommitted = yield* Deferred.make<void>();
3534
+ const releaseFirst = yield* Deferred.make<void>();
3535
+ const layerPausingOutbox = Layer.effect(
3536
+ EffectOutbox.EffectOutboxV2,
3537
+ Effect.gen(function* () {
3538
+ const delegate = yield* EffectOutbox.EffectOutboxV2;
3539
+ return EffectOutbox.EffectOutboxV2.of({
3540
+ ...delegate,
3541
+ notifyAvailable: (count) =>
3542
+ Deferred.succeed(firstCommitted, undefined).pipe(
3543
+ Effect.andThen(Deferred.await(releaseFirst)),
3544
+ Effect.andThen(delegate.notifyAvailable(count)),
3545
+ ),
3546
+ });
3547
+ }),
3548
+ ).pipe(Layer.provide(layerEffectOutboxProvided));
3549
+ const layerEventSink = EventSink.layerFromStores.pipe(
3550
+ Layer.provide(
3551
+ Layer.mergeAll(
3552
+ layerStoresProvided,
3553
+ layerPausingOutbox,
3554
+ layerCommandReceiptStoreProvided,
3555
+ ProjectStore.layer.pipe(Layer.provide(layerDatabase)),
3556
+ TurnItemPositionStore.layer.pipe(Layer.provide(layerDatabase)),
3557
+ ),
3558
),
3318
- ),
3319
- );
3559
+ );
3560
3321
- yield* Effect.gen(function* () {
3322
- const eventSink = yield* EventSink.EventSinkV2;
3323
- const now = yield* DateTime.now;
3324
- const first = makeThread(ThreadId.make("thread:foundation-publish-order:first"), now);
3325
- const second = makeThread(ThreadId.make("thread:foundation-publish-order:second"), now);
3326
- const published = yield* eventSink
3327
- .stream({ afterSequence: yield* eventSink.latestSequence() })
3328
- .pipe(Stream.take(2), Stream.runCollect, Effect.forkScoped({ startImmediately: true }));
3329
-
3330
- const firstWrite = yield* eventSink
3331
- .writeWithEffects({
3332
- events: [
3333
- threadCreatedEvent({ id: "event:foundation-publish-order:first", thread: first, now }),
3334
- ],
3335
- effects: [
3336
- {
3337
- id: "effect:foundation-publish-order:first",
3338
- commandId: CommandId.make("command:foundation-publish-order:first"),
3339
- threadId: first.id,
3340
- request: { type: "terminal.cleanup" },
3341
- },
3342
- ],
3343
- })
3344
- .pipe(Effect.forkScoped);
3345
- yield* Deferred.await(firstCommitted);
3346
- const secondWrite = yield* eventSink
3347
- .write({
3348
- events: [
3349
- threadCreatedEvent({
3350
- id: "event:foundation-publish-order:second",
3351
- thread: second,
3352
- now,
3353
- }),
3354
- ],
3355
- })
3356
- .pipe(
3357
- Effect.provideService(Scheduler.MaxOpsBeforeYield, Number.POSITIVE_INFINITY),
3358
- Effect.forkScoped,
3359
- );
3360
- yield* Effect.yieldNow;
3361
- yield* Deferred.succeed(releaseFirst, undefined);
3362
- yield* Fiber.join(firstWrite);
3363
- yield* Fiber.join(secondWrite);
3561
+ yield* Effect.gen(function* () {
3562
+ const eventSink = yield* EventSink.EventSinkV2;
3563
+ const now = yield* DateTime.now;
3564
+ const first = makeThread(ThreadId.make("thread:foundation-publish-order:first"), now);
3565
+ const second = makeThread(ThreadId.make("thread:foundation-publish-order:second"), now);
3566
+ const published = yield* eventSink
3567
+ .stream({ afterSequence: yield* eventSink.latestSequence() })
3568
+ .pipe(Stream.take(2), Stream.runCollect, Effect.forkScoped({ startImmediately: true }));
3569
3365
- const sequences = Array.from(yield* Fiber.join(published), (stored) => stored.sequence);
3366
- assert.deepEqual(
3367
- sequences,
3368
- [...sequences].sort((left, right) => left - right),
3369
- );
3370
- }).pipe(Effect.provide(layerEventSink));
3371
- }).pipe(Effect.provide(layerDatabase)),
3570
+ const firstWrite = yield* eventSink
3571
+ .writeWithEffects({
3572
+ events: [
3573
+ threadCreatedEvent({
3574
+ id: "event:foundation-publish-order:first",
3575
+ thread: first,
3576
+ now,
3577
+ }),
3578
+ ],
3579
+ effects: [
3580
+ {
3581
+ id: "effect:foundation-publish-order:first",
3582
+ commandId: CommandId.make("command:foundation-publish-order:first"),
3583
+ threadId: first.id,
3584
+ request: { type: "terminal.cleanup" },
3585
+ },
3586
+ ],
3587
+ })
3588
+ .pipe(
3589
+ (write) => (mode === "direct" ? write : eventSink.withProviderBatch(write)),
3590
+ Effect.forkScoped,
3591
+ );
3592
+ yield* Deferred.await(firstCommitted);
3593
+ const secondWrite = yield* eventSink
3594
+ .write({
3595
+ events: [
3596
+ threadCreatedEvent({
3597
+ id: "event:foundation-publish-order:second",
3598
+ thread: second,
3599
+ now,
3600
+ }),
3601
+ ],
3602
+ })
3603
+ .pipe(
3604
+ Effect.provideService(Scheduler.MaxOpsBeforeYield, Number.POSITIVE_INFINITY),
3605
+ Effect.forkScoped,
3606
+ );
3607
+ yield* Effect.yieldNow;
3608
+ const interrupted =
3609
+ mode === "cancel-provider-batch"
3610
+ ? yield* Fiber.interrupt(firstWrite).pipe(Effect.forkScoped({ startImmediately: true }))
3611
+ : undefined;
3612
+ yield* Deferred.succeed(releaseFirst, undefined);
3613
+ if (interrupted !== undefined) yield* Fiber.join(interrupted);
3614
+ else yield* Fiber.join(firstWrite);
3615
+ yield* Fiber.join(secondWrite);
3616
+
3617
+ const sequences = Array.from(yield* Fiber.join(published), (stored) => stored.sequence);
3618
+ assert.deepEqual(
3619
+ sequences,
3620
+ [...sequences].sort((left, right) => left - right),
3621
+ );
3622
+ }).pipe(Effect.provide(layerEventSink));
3623
+ }).pipe(Effect.provide(layerDatabase)),
3624
);
3625
3626
it.effect.each(["events", "receipts"] as const)(
apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts
new
+472
@@ -0,0 +1,472 @@
1
+import { assert, it } from "@effect/vitest";
2
+import { MessageId, ProviderDriverKind, ProviderTurnId, ThreadId } from "@t3tools/contracts";
3
+import * as KeyedLock from "@t3tools/shared/KeyedLock";
4
+import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
5
+import * as Cause from "effect/Cause";
6
+import * as DateTime from "effect/DateTime";
7
+import * as Deferred from "effect/Deferred";
8
+import * as Effect from "effect/Effect";
9
+import * as Exit from "effect/Exit";
10
+import * as Fiber from "effect/Fiber";
11
+import * as Queue from "effect/Queue";
12
+import * as Stream from "effect/Stream";
13
+import * as TestClock from "effect/testing/TestClock";
14
+import * as ProviderCommitBatch from "./ProviderCommitBatch.ts";
15
+
16
+const driver = ProviderDriverKind.make("codex");
17
+function event(index: number, text = `delta:${index}`): ProviderAdapter.ProviderAdapterV2Event {
18
+ const threadId = ThreadId.make("single-thread");
19
+ const now = DateTime.makeUnsafe("2026-11-03T00:00:00Z");
20
+ return {
21
+ type: "message.updated",
22
+ driver,
23
+ message: {
24
+ id: MessageId.make("single-message"),
25
+ threadId,
26
+ runId: null,
27
+ nodeId: null,
28
+ createdBy: "agent",
29
+ creationSource: "provider",
30
+ role: "assistant",
31
+ text,
32
+ attachments: [],
33
+ streaming: true,
34
+ createdAt: now,
35
+ updatedAt: now,
36
+ },
37
+ };
38
+}
39
+const control: ProviderAdapter.ProviderAdapterV2Event = {
40
+ type: "background_work.updated",
41
+ driver,
42
+ providerTurnId: ProviderTurnId.make("turn"),
43
+ pending: false,
44
+};
45
+
46
+const recordBatches = Effect.fn("test.recordProviderBatches")(function* (
47
+ beforeCommit: Effect.Effect<void> = Effect.void,
48
+) {
49
+ const output = yield* Queue.unbounded<ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event>>();
50
+ let current: ProviderAdapter.ProviderAdapterV2Event[] = [];
51
+ const coordinator = yield* ProviderCommitBatch.make(
52
+ (work) =>
53
+ Effect.gen(function* () {
54
+ yield* beforeCommit;
55
+ current = [];
56
+ yield* work;
57
+ if (current.length > 0) yield* Queue.offer(output, current);
58
+ }),
59
+ true,
60
+ );
61
+ const process = (raw: ProviderAdapter.ProviderAdapterV2Event) =>
62
+ raw.type === "background_work.updated"
63
+ ? Queue.offer(output, [raw]).pipe(Effect.asVoid)
64
+ : Effect.sync(() => {
65
+ current.push(raw);
66
+ });
67
+ return { ...coordinator, process, output };
68
+});
69
+
70
+it.effect("publishes preceding snapshots before a control", () =>
71
+ Effect.gen(function* () {
72
+ const recorder = yield* recordBatches();
73
+ const input = yield* Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event>();
74
+ const consumer = yield* recorder
75
+ .consume(Stream.fromQueue(input), recorder.process, Effect.succeed(true))
76
+ .pipe(Effect.forkScoped);
77
+ yield* Queue.offer(input, event(1));
78
+ assert.deepEqual(yield* Queue.take(recorder.output), [event(1)]);
79
+ yield* Queue.offer(input, event(2));
80
+ yield* Queue.offer(input, control);
81
+ assert.deepEqual(yield* Queue.take(recorder.output), [event(2)]);
82
+ assert.deepEqual(yield* Queue.take(recorder.output), [control]);
83
+ yield* Fiber.join(consumer);
84
+ }).pipe(Effect.scoped),
85
+);
86
+
87
+it.effect("publishes isolated snapshots without advancing the clock", () =>
88
+ Effect.gen(function* () {
89
+ const recorder = yield* recordBatches();
90
+ const input = yield* Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event>();
91
+ const pulled = yield* Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event>();
92
+ yield* recorder
93
+ .consume(
94
+ Stream.fromQueue(input).pipe(Stream.tap((raw) => Queue.offer(pulled, raw))),
95
+ recorder.process,
96
+ Effect.succeed(false),
97
+ )
98
+ .pipe(Effect.forkScoped);
99
+ for (const raw of [event(1), event(2)]) {
100
+ yield* Queue.offer(input, raw);
101
+ yield* Queue.take(pulled);
102
+ yield* TestClock.adjust(0);
103
+ const published = yield* Queue.poll(recorder.output);
104
+ assert.equal(published._tag, "Some");
105
+ if (published._tag === "Some") assert.deepEqual(published.value, [raw]);
106
+ }
107
+ }).pipe(Effect.scoped),
108
+);
109
+
110
+it.effect("bounds transactions by count and bytes, including the final partial group", () =>
111
+ Effect.gen(function* () {
112
+ const workloads = [
113
+ { events: Array.from({ length: 129 }, (_, index) => event(index)) },
114
+ { events: [event(0), event(1, "x".repeat(1024 * 1024)), event(2)] },
115
+ {
116
+ events: [event(0, "x".repeat(600_000)), event(1, "y".repeat(600_000)), event(2)],
117
+ },
118
+ ];
119
+ for (const { events } of workloads) {
120
+ const recorder = yield* recordBatches();
121
+ const pulled = yield* Deferred.make<void>();
122
+ let arrivals = 0;
123
+ const consumer = yield* recorder
124
+ .consume(
125
+ Stream.fromArray(events).pipe(
126
+ Stream.tap(() =>
127
+ ++arrivals === events.length ? Deferred.succeed(pulled, undefined) : Effect.void,
128
+ ),
129
+ ),
130
+ recorder.process,
131
+ Effect.succeed(false),
132
+ )
133
+ .pipe(Effect.forkScoped({ startImmediately: true }));
134
+ yield* Deferred.await(pulled);
135
+ yield* TestClock.adjust(0);
136
+ yield* recorder.barrier(Effect.void);
137
+ yield* Fiber.join(consumer);
138
+ const batches = yield* Queue.takeAll(recorder.output);
139
+ assert.isTrue(batches.every((batch) => batch.length <= 64));
140
+ if (events.length === 129) assert.isTrue(batches.some((batch) => batch.length > 1));
141
+ assert.isTrue(
142
+ batches.every(
143
+ (batch) =>
144
+ batch.length === 1 ||
145
+ batch.reduce((bytes, raw) => bytes + Buffer.byteLength(JSON.stringify(raw)), 0) <=
146
+ 1024 * 1024,
147
+ ),
148
+ );
149
+ assert.deepEqual(batches.flat(), events);
150
+ }
151
+ }).pipe(Effect.scoped),
152
+);
153
+
154
+it.effect("keeps updates immediate and ordered when batching is disabled", () =>
155
+ Effect.gen(function* () {
156
+ const seen: ProviderAdapter.ProviderAdapterV2Event[] = [];
157
+ const coordinator = yield* ProviderCommitBatch.make(() => Effect.die("must not batch"), false);
158
+ yield* coordinator.consume(
159
+ Stream.make(event(1), control, event(2)),
160
+ (raw) =>
161
+ Effect.sync(() => {
162
+ seen.push(raw);
163
+ }),
164
+ Effect.succeed(false),
165
+ );
166
+ assert.deepEqual(seen, [event(1), control, event(2)]);
167
+ }).pipe(Effect.scoped),
168
+);
169
+
170
+it.effect.each([
171
+ { streams: 1, updates: 10 },
172
+ { streams: 10, updates: 1 },
173
+ { streams: 10, updates: 10 },
174
+])(
175
+ "shares bounded transactions across $streams streams with $updates updates each",
176
+ ({ streams, updates }) =>
177
+ Effect.gen(function* () {
178
+ const coordinator = yield* recordBatches();
179
+ const inputs = yield* Effect.forEach(Array.from({ length: streams }), () =>
180
+ Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event>(),
181
+ );
182
+ const pulled = yield* Deferred.make<void>();
183
+ let arrivals = 0;
184
+ const consumers = yield* Effect.forEach(inputs, (input) =>
185
+ coordinator
186
+ .consume(
187
+ Stream.fromQueue(input).pipe(
188
+ Stream.tap(() => {
189
+ arrivals++;
190
+ return arrivals === streams * updates
191
+ ? Deferred.succeed(pulled, undefined)
192
+ : Effect.void;
193
+ }),
194
+ ),
195
+ coordinator.process,
196
+ Effect.succeed(true),
197
+ )
198
+ .pipe(Effect.forkScoped({ startImmediately: true })),
199
+ );
200
+ for (let n = 0; n < updates; n++) {
201
+ for (let stream = 0; stream < streams; stream++)
202
+ yield* Queue.offer(inputs[stream]!, event(stream * updates + n));
203
+ }
204
+ yield* Deferred.await(pulled);
205
+ yield* TestClock.adjust(0);
206
+
207
+ yield* coordinator.barrier(Effect.void);
208
+ const batches: Array<ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event>> = [];
209
+ let count = 0;
210
+ while (count < streams * updates) {
211
+ const batch = yield* Queue.take(coordinator.output);
212
+ batches.push(batch);
213
+ count += batch.length;
214
+ }
215
+ assert.deepEqual(
216
+ batches.map((batch) => batch.length),
217
+ streams * updates > 64 ? [64, 36] : [10],
218
+ );
219
+ assert.equal(
220
+ new Set(
221
+ batches
222
+ .flat()
223
+ .flatMap((raw) => (raw.type === "message.updated" ? [raw.message.text] : [])),
224
+ ).size,
225
+ streams * updates,
226
+ );
227
+ for (const input of inputs) yield* Queue.offer(input, control);
228
+ yield* Effect.forEach(consumers, Fiber.join);
229
+ }).pipe(Effect.scoped),
230
+);
231
+
232
+it.effect("does not hold the shared worker while one consumer checks background work", () =>
233
+ Effect.gen(function* () {
234
+ const checking = yield* Deferred.make<void>();
235
+ const release = yield* Deferred.make<void>();
236
+ const coordinator = yield* ProviderCommitBatch.make((work) => work, true);
237
+ const consumer = yield* coordinator
238
+ .consume(
239
+ Stream.make(control),
240
+ () => Effect.void,
241
+ Deferred.succeed(checking, undefined).pipe(
242
+ Effect.andThen(Deferred.await(release)),
243
+ Effect.as(true),
244
+ ),
245
+ )
246
+ .pipe(Effect.forkScoped({ startImmediately: true }));
247
+ yield* Deferred.await(checking);
248
+ assert.equal(
249
+ yield* coordinator.barrier(Effect.succeed("another run can commit")),
250
+ "another run can commit",
251
+ );
252
+ yield* Deferred.succeed(release, undefined);
253
+ yield* Fiber.join(consumer);
254
+ }).pipe(Effect.scoped),
255
+);
256
+
257
+it.effect.each(["job", "commit"] as const)(
258
+ "isolates a %s failure at its transaction boundary",
259
+ (mode) =>
260
+ Effect.gen(function* () {
261
+ const coordinator = yield* ProviderCommitBatch.make(
262
+ (work) =>
263
+ mode === "commit"
264
+ ? work.pipe(
265
+ Effect.andThen(
266
+ Effect.fail(new ProviderCommitBatch.CommitBatchError({ cause: "COMMIT failed" })),
267
+ ),
268
+ )
269
+ : work,
270
+ true,
271
+ );
272
+ const inputs = yield* Effect.forEach([0, 1], () =>
273
+ Queue.unbounded<ProviderAdapter.ProviderAdapterV2Event>(),
274
+ );
275
+ const consumers = yield* Effect.forEach(inputs, (input, index) =>
276
+ coordinator
277
+ .consume(
278
+ Stream.fromQueue(input),
279
+ () => (mode === "job" && index === 1 ? Effect.fail("invalid update") : Effect.void),
280
+ Effect.succeed(true),
281
+ )
282
+ .pipe(Effect.exit, Effect.forkScoped({ startImmediately: true })),
283
+ );
284
+ for (const input of inputs) yield* Queue.offer(input, event(0));
285
+ yield* TestClock.adjust(0);
286
+ yield* coordinator.barrier(Effect.void);
287
+ assert.isTrue(Exit.isFailure(yield* Fiber.join(consumers[1]!)));
288
+ if (mode === "commit") assert.isTrue(Exit.isFailure(yield* Fiber.join(consumers[0]!)));
289
+ else {
290
+ assert.isUndefined(consumers[0]!.pollUnsafe());
291
+ yield* Queue.offer(inputs[0]!, control);
292
+ assert.isTrue(Exit.isSuccess(yield* Fiber.join(consumers[0]!)));
293
+ }
294
+ assert.equal(yield* coordinator.barrier(Effect.succeed("recovered")), "recovered");
295
+ assert.deepEqual(
296
+ yield* coordinator.barrier(Effect.fail("original error")).pipe(Effect.exit),
297
+ Exit.fail("original error"),
298
+ );
299
+ }).pipe(Effect.scoped),
300
+);
301
+
302
+it.effect("keeps a transaction failure when rolling back one failed snapshot", () =>
303
+ Effect.gen(function* () {
304
+ const transactionError = new ProviderCommitBatch.CommitBatchError({ cause: "ROLLBACK failed" });
305
+ const coordinator = yield* ProviderCommitBatch.make(
306
+ (work) => work.pipe(Effect.mapError(() => transactionError)),
307
+ true,
308
+ );
309
+ const exit = yield* coordinator
310
+ .consume(
311
+ Stream.concat(Stream.make(event(1)), Stream.never),
312
+ () => Effect.fail("invalid snapshot"),
313
+ Effect.succeed(false),
314
+ )
315
+ .pipe(Effect.exit);
316
+ assert.isTrue(Exit.isFailure(exit));
317
+ if (Exit.isFailure(exit)) assert.strictEqual(Cause.squash(exit.cause), transactionError);
318
+ }).pipe(Effect.scoped),
319
+);
320
+
321
+it.effect(
322
+ "allows a command holding the thread lock to commit while a provider control waits for it",
323
+ () =>
324
+ Effect.gen(function* () {
325
+ const coordinator = yield* ProviderCommitBatch.make((work) => work, true);
326
+ const locks = yield* KeyedLock.make<ThreadId>();
327
+ const threadId = ThreadId.make("locked-thread");
328
+ const held = yield* Deferred.make<void>();
329
+ const waiting = yield* Deferred.make<void>();
330
+ const committed = yield* Deferred.make<void>();
331
+ const command = yield* locks
332
+ .withLock(
333
+ threadId,
334
+ Deferred.succeed(held, undefined).pipe(
335
+ Effect.andThen(Deferred.await(waiting)),
336
+ Effect.andThen(coordinator.barrier(Deferred.succeed(committed, undefined))),
337
+ ),
338
+ )
339
+ .pipe(Effect.forkScoped({ startImmediately: true }));
340
+ yield* Deferred.await(held);
341
+ const consumer = yield* coordinator
342
+ .consume(
343
+ Stream.make(control),
344
+ () =>
345
+ Deferred.succeed(waiting, undefined).pipe(
346
+ Effect.andThen(locks.withLock(threadId, Effect.void)),
347
+ ),
348
+ Effect.succeed(true),
349
+ )
350
+ .pipe(Effect.forkScoped({ startImmediately: true }));
351
+ yield* Deferred.await(waiting);
352
+ yield* TestClock.adjust(0);
353
+ assert.isTrue(yield* Deferred.isDone(committed));
354
+ yield* Fiber.join(command);
355
+ yield* Fiber.join(consumer);
356
+ }).pipe(Effect.scoped),
357
+);
358
+
359
+it.effect.each([false, true])(
360
+ "batches work accumulated behind a busy writer with cancellation=%s",
361
+ (cancel) =>
362
+ Effect.gen(function* () {
363
+ const started = yield* Deferred.make<void>();
364
+ const release = yield* Deferred.make<void>();
365
+ let first = true;
366
+ const recorder = yield* recordBatches(
367
+ Effect.gen(function* () {
368
+ if (!first) return;
369
+ first = false;
370
+ yield* Deferred.succeed(started, undefined);
371
+ yield* Deferred.await(release);
372
+ }),
373
+ );
374
+ const leading = yield* recorder
375
+ .consume(Stream.make(event(-1)), recorder.process, Effect.succeed(false))
376
+ .pipe(Effect.forkScoped({ startImmediately: true }));
377
+ yield* Deferred.await(started);
378
+ const pulled = yield* Deferred.make<void>();
379
+ let arrivals = 0;
380
+ const consumers = yield* Effect.forEach(Array.from({ length: 10 }), (_, index) =>
381
+ recorder
382
+ .consume(
383
+ Stream.make(event(index)).pipe(
384
+ Stream.tap(() =>
385
+ ++arrivals === 10 ? Deferred.succeed(pulled, undefined) : Effect.void,
386
+ ),
387
+ ),
388
+ recorder.process,
389
+ Effect.succeed(false),
390
+ )
391
+ .pipe(Effect.forkScoped({ startImmediately: true })),
392
+ );
393
+ yield* Deferred.await(pulled);
394
+ yield* TestClock.adjust(0);
395
+ if (cancel) yield* Fiber.interrupt(consumers[0]!);
396
+ assert.equal(yield* Queue.size(recorder.output), 0);
397
+ yield* Deferred.succeed(release, undefined);
398
+ yield* Fiber.join(leading);
399
+ yield* Effect.forEach(consumers.slice(cancel ? 1 : 0), Fiber.join);
400
+ assert.deepEqual(yield* Queue.takeAll(recorder.output), [
401
+ [event(-1)],
402
+ Array.from({ length: cancel ? 9 : 10 }, (_, index) => event(index + (cancel ? 1 : 0))),
403
+ ]);
404
+ }).pipe(Effect.scoped),
405
+);
406
+
407
+it.effect.each(["failure", "stop"] as const)(
408
+ "flushes the prefix and respects a source %s barrier",
409
+ (mode) =>
410
+ Effect.gen(function* () {
411
+ const seen: ProviderAdapter.ProviderAdapterV2Event[] = [];
412
+ const coordinator = yield* ProviderCommitBatch.make((work) => work, true);
413
+ const source =
414
+ mode === "failure"
415
+ ? Stream.concat(Stream.make(event(1), event(2)), Stream.fail("provider failed"))
416
+ : Stream.make(event(1), control, event(2));
417
+ const exit = yield* coordinator
418
+ .consume(
419
+ source,
420
+ (raw) =>
421
+ Effect.sync(() => {
422
+ seen.push(raw);
423
+ }),
424
+ Effect.succeed(mode === "stop"),
425
+ )
426
+ .pipe(Effect.exit);
427
+ assert.deepEqual(seen, mode === "failure" ? [event(1), event(2)] : [event(1), control]);
428
+ assert.equal(Exit.isFailure(exit), mode === "failure");
429
+ assert.equal(
430
+ yield* coordinator.barrier(Effect.succeed("still available")),
431
+ "still available",
432
+ );
433
+ }).pipe(Effect.scoped),
434
+);
435
+
436
+it.effect("does not force a separate commit for every consumer that ends", () =>
437
+ Effect.gen(function* () {
438
+ const pulled = yield* Deferred.make<void>();
439
+ let arrivals = 0;
440
+ const coordinator = yield* recordBatches();
441
+ for (let index = 0; index < 10; index++) {
442
+ yield* coordinator
443
+ .consume(
444
+ Stream.make(event(index)).pipe(
445
+ Stream.tap(() =>
446
+ ++arrivals === 10 ? Deferred.succeed(pulled, undefined) : Effect.void,
447
+ ),
448
+ ),
449
+ coordinator.process,
450
+ Effect.succeed(false),
451
+ )
452
+ .pipe(Effect.forkScoped({ startImmediately: true }));
453
+ }
454
+ yield* Deferred.await(pulled);
455
+ yield* TestClock.adjust(0);
456
+ yield* coordinator.barrier(Effect.void);
457
+ assert.deepEqual(
458
+ (yield* Queue.takeAll(coordinator.output)).map((batch) => batch.length),
459
+ [10],
460
+ );
461
+ }).pipe(Effect.scoped),
462
+);
463
+
464
+it.effect("settles calls made after the coordinator's scope closes", () =>
465
+ Effect.gen(function* () {
466
+ const coordinator = yield* ProviderCommitBatch.make((work) => work, true).pipe(Effect.scoped);
467
+ const exit = yield* coordinator
468
+ .barrier(Effect.die("must not execute after shutdown"))
469
+ .pipe(Effect.exit);
470
+ assert.isTrue(Exit.isFailure(exit) && Cause.hasInterruptsOnly(exit.cause));
471
+ }),
472
+);
apps/server/src/orchestration-v2/ProviderCommitBatch.ts
new
+303
@@ -0,0 +1,303 @@
1
+import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
2
+import * as Cause from "effect/Cause";
3
+import * as Clock from "effect/Clock";
4
+import * as Config from "effect/Config";
5
+import * as Effect from "effect/Effect";
6
+import * as Context from "effect/Context";
7
+import * as Deferred from "effect/Deferred";
8
+import * as Exit from "effect/Exit";
9
+import * as Queue from "effect/Queue";
10
+import * as Schema from "effect/Schema";
11
+import * as Stream from "effect/Stream";
12
+
13
+const MAX_EVENTS = 64;
14
+const MAX_BYTES = 1024 * 1024;
15
+const encodeQueue = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown));
16
+
17
+export const enabled = Config.Boolean("T3CODE_PROVIDER_COMMIT_BATCHING").pipe(
18
+ Config.withDefault(true),
19
+ Effect.orDie,
20
+);
21
+
22
+type Event = ProviderAdapter.ProviderAdapterV2Event;
23
+const isSnapshot = (event: Event) =>
24
+ event.type === "node.updated" ||
25
+ event.type === "message.updated" ||
26
+ event.type === "turn_item.updated" ||
27
+ event.type === "plan.updated";
28
+
29
+export class CommitBatchError extends Schema.TaggedError<CommitBatchError>()("CommitBatchError", {
30
+ cause: Schema.Defect(),
31
+}) {}
32
+export const isCommitBatchError = Schema.is(CommitBatchError);
33
+
34
+interface Job {
35
+ readonly owner: { active: boolean };
36
+ readonly queuedAt: number;
37
+ readonly snapshot: boolean;
38
+ readonly bytes: number;
39
+ readonly execute: Effect.Effect<void, CommitBatchError>;
40
+ readonly settle: (
41
+ exit: Exit.Exit<void, CommitBatchError>,
42
+ preserveFailure: boolean,
43
+ ) => Effect.Effect<void>;
44
+ readonly cancel: Effect.Effect<void>;
45
+}
46
+interface Batch {
47
+ readonly events: ReadonlyArray<Job>;
48
+ readonly bytes: number;
49
+ readonly reason: "ready" | "limit" | "control";
50
+}
51
+
52
+function collect(queue: Queue.Queue<Job>) {
53
+ return Stream.fromEffectRepeat(Queue.takeBetween(queue, 1, MAX_EVENTS)).pipe(
54
+ Stream.flatMap((ready) => {
55
+ const batches: Array<Batch> = [];
56
+ let events: Array<Job> = [];
57
+ let bytes = 0;
58
+ const flush = (reason: Batch["reason"]) => {
59
+ if (events.length > 0) batches.push({ events, bytes, reason });
60
+ events = [];
61
+ bytes = 0;
62
+ };
63
+ for (const job of ready) {
64
+ if (!job.snapshot) {
65
+ flush("control");
66
+ batches.push({ events: [job], bytes: 0, reason: "control" });
67
+ } else {
68
+ if (events.length > 0 && bytes + job.bytes > MAX_BYTES) flush("limit");
69
+ events.push(job);
70
+ bytes += job.bytes;
71
+ if (events.length >= MAX_EVENTS || bytes >= MAX_BYTES) flush("limit");
72
+ }
73
+ }
74
+ flush("ready");
75
+ return Stream.fromArray(batches);
76
+ }),
77
+ );
78
+}
79
+
80
+const CurrentWorker = Context.Reference<number | undefined>(
81
+ "t3/ProviderCommitBatch/CurrentWorker",
82
+ {
83
+ defaultValue: () => undefined,
84
+ },
85
+);
86
+
87
+export const make = Effect.fn("ProviderCommitBatch.make")(function* (
88
+ commit: (work: Effect.Effect<void, CommitBatchError>) => Effect.Effect<void, CommitBatchError>,
89
+ enabled: boolean,
90
+ isolate: (
91
+ work: Effect.Effect<void, CommitBatchError>,
92
+ ) => Effect.Effect<void, CommitBatchError> = (work) => work,
93
+) {
94
+ const queue = yield* Queue.bounded<Job>(MAX_EVENTS);
95
+ const pending = new Set<Job>();
96
+ let closed = false;
97
+ const shutdown = Effect.gen(function* () {
98
+ closed = true;
99
+ yield* Queue.shutdown(queue);
100
+ for (const job of pending) yield* job.cancel;
101
+ });
102
+ yield* Effect.addFinalizer(() => shutdown);
103
+ // These helpers run once per raw event; the shared transaction owns the tracing span.
104
+ const enqueue = Effect.fnUntraced(function* <A, E, R>(
105
+ effect: Effect.Effect<A, E, R>,
106
+ token: { active: boolean },
107
+ snapshot: boolean,
108
+ bytes: number,
109
+ ) {
110
+ const caller = yield* Effect.context<R>();
111
+ const reply = yield* Deferred.make<A, E | CommitBatchError>();
112
+ let ownFailure: Cause.Cause<E> | undefined;
113
+ let result: A;
114
+ const job: Job = {
115
+ owner: token,
116
+ queuedAt: yield* Clock.currentTimeMillis,
117
+ snapshot,
118
+ bytes,
119
+ execute: effect.pipe(
120
+ Effect.tapCause((cause) =>
121
+ Effect.sync(() => {
122
+ ownFailure = cause;
123
+ }),
124
+ ),
125
+ Effect.mapError((cause) => new CommitBatchError({ cause })),
126
+ Effect.provideContext(caller),
127
+ Effect.tap((value) =>
128
+ Effect.sync(() => {
129
+ result = value;
130
+ }),
131
+ ),
132
+ Effect.asVoid,
133
+ ),
134
+ settle: Effect.fnUntraced(function* (
135
+ exit: Exit.Exit<void, CommitBatchError>,
136
+ preserveFailure: boolean,
137
+ ) {
138
+ pending.delete(job);
139
+ if (Exit.isFailure(exit)) {
140
+ token.active = false;
141
+ yield* Deferred.failCause(
142
+ reply,
143
+ preserveFailure && ownFailure !== undefined ? ownFailure : exit.cause,
144
+ );
145
+ } else yield* Deferred.succeed(reply, result);
146
+ }),
147
+ cancel: Effect.gen(function* () {
148
+ pending.delete(job);
149
+ yield* Deferred.interrupt(reply);
150
+ }),
151
+ };
152
+ yield* Effect.uninterruptibleMask((restore) =>
153
+ Effect.gen(function* () {
154
+ pending.add(job);
155
+ const accepted = yield* restore(Queue.offer(queue, job)).pipe(
156
+ Effect.onError(() => job.cancel),
157
+ );
158
+ if (!accepted) yield* job.cancel;
159
+ }),
160
+ );
161
+ return Deferred.await(reply);
162
+ });
163
+ if (enabled) {
164
+ yield* collect(queue).pipe(
165
+ Stream.runForEach((batch) =>
166
+ Effect.uninterruptibleMask((restore) =>
167
+ Effect.gen(function* () {
168
+ const jobs = batch.events.filter((job) => job.owner.active);
169
+ for (const job of batch.events) if (!job.owner.active) yield* job.cancel;
170
+ if (jobs.length === 0) return;
171
+ yield* Effect.annotateCurrentSpan({
172
+ "orchestration_v2.work_item_count": jobs.length,
173
+ "orchestration_v2.consumer_count": new Set(jobs.map((job) => job.owner)).size,
174
+ "orchestration_v2.raw_json_bytes": batch.bytes,
175
+ "orchestration_v2.batch_flush_reason": batch.reason,
176
+ "orchestration_v2.backlog_batching": true,
177
+ "orchestration_v2.batch_wait_ms":
178
+ (yield* Clock.currentTimeMillis) - Math.min(...jobs.map((job) => job.queuedAt)),
179
+ });
180
+ const outcomes = new Map<Job, Exit.Exit<void, CommitBatchError>>();
181
+ const work = Effect.forEach(
182
+ jobs,
183
+ (job) =>
184
+ Effect.gen(function* () {
185
+ if (!job.owner.active) return yield* job.cancel;
186
+ const outcome = yield* (
187
+ jobs.length > 1 && job.bytes > 0 ? isolate(job.execute) : job.execute
188
+ ).pipe(Effect.exit);
189
+ outcomes.set(job, outcome);
190
+ if (Exit.isFailure(outcome)) {
191
+ job.owner.active = false;
192
+ if (jobs.length === 1) return yield* Effect.failCause(outcome.cause);
193
+ }
194
+ }),
195
+ { discard: true },
196
+ );
197
+ const exit = yield* restore(
198
+ (jobs.every((job) => job.bytes === 0) ? work : commit(work)).pipe(
199
+ Effect.provideService(CurrentWorker, yield* Effect.fiberId),
200
+ ),
201
+ ).pipe(Effect.exit);
202
+ for (const job of jobs) {
203
+ const outcome = outcomes.get(job) ?? Exit.void;
204
+ yield* job.settle(
205
+ Exit.isSuccess(exit) ? outcome : exit,
206
+ Exit.isSuccess(exit) ||
207
+ (Exit.isFailure(exit) &&
208
+ Exit.isFailure(outcome) &&
209
+ Cause.squash(exit.cause) === Cause.squash(outcome.cause)),
210
+ );
211
+ }
212
+ }),
213
+ ).pipe(Effect.withSpan("orchestrationV2.ProviderCommitBatch.flush")),
214
+ ),
215
+ Effect.ensuring(shutdown),
216
+ Effect.forkScoped,
217
+ );
218
+ }
219
+ const barrier = Effect.fnUntraced(function* <A, E, R>(effect: Effect.Effect<A, E, R>) {
220
+ if (closed) return yield* Effect.interrupt;
221
+ if (!enabled || pending.size === 0 || (yield* CurrentWorker) === (yield* Effect.fiberId))
222
+ return yield* effect;
223
+ const token = { active: true };
224
+ return yield* enqueue(Effect.exit(effect), token, false, 0).pipe(
225
+ Effect.flatten,
226
+ Effect.orDie,
227
+ Effect.flatten,
228
+ Effect.ensuring(
229
+ Effect.sync(() => {
230
+ token.active = false;
231
+ }),
232
+ ),
233
+ );
234
+ });
235
+ const consume = Effect.fnUntraced(function* <E, R, E2, R2>(
236
+ source: Stream.Stream<Event, E, R>,
237
+ process: (event: Event) => Effect.Effect<void, E2, R2>,
238
+ after: Effect.Effect<boolean, E2, R2>,
239
+ canBatch: Effect.Effect<boolean, never, R2> = Effect.succeed(true),
240
+ ) {
241
+ if (!enabled)
242
+ return yield* source.pipe(
243
+ Stream.mapEffect((event) => process(event).pipe(Effect.andThen(after))),
244
+ Stream.takeUntil(Boolean),
245
+ Stream.runDrain,
246
+ );
247
+ const token = { active: true };
248
+ const signals = source.pipe(
249
+ Stream.map((event) => ({ _tag: "Event" as const, event })),
250
+ Stream.catchCause((cause) => Stream.succeed({ _tag: "Failure" as const, cause })),
251
+ Stream.concat(Stream.succeed({ _tag: "End" as const })),
252
+ );
253
+ let resume: Effect.Effect<void> = Effect.void;
254
+ yield* signals.pipe(
255
+ Stream.mapEffect((signal) =>
256
+ Effect.gen(function* () {
257
+ yield* resume;
258
+ const processed = yield* Deferred.make<void>();
259
+ const event = signal._tag === "Event" ? signal.event : undefined;
260
+ // EOF only acknowledges prior work; it can share a commit with other consumers.
261
+ const snapshot =
262
+ signal._tag === "End" ||
263
+ (event !== undefined && isSnapshot(event) && (yield* canBatch));
264
+ const bytes =
265
+ snapshot && event !== undefined ? Buffer.byteLength(yield* encodeQueue(event)) : 0;
266
+ const work: Effect.Effect<boolean, E | E2, R2> =
267
+ signal._tag === "Failure"
268
+ ? Effect.failCause(signal.cause)
269
+ : event === undefined
270
+ ? Effect.succeed(true)
271
+ : process(event).pipe(Effect.as(false));
272
+ const reply = snapshot
273
+ ? yield* enqueue(work, token, true, bytes)
274
+ : (yield* enqueue(Effect.void, token, false, 0)).pipe(Effect.andThen(work));
275
+ if (!snapshot) resume = Deferred.await(processed);
276
+ return { reply, processed, check: !snapshot };
277
+ }),
278
+ ),
279
+ Stream.buffer({ capacity: MAX_EVENTS }),
280
+ Stream.mapEffect(({ reply, processed, check }) =>
281
+ reply.pipe(
282
+ Effect.flatMap((end) =>
283
+ end ? Effect.succeed(true) : check ? after : Effect.succeed(false),
284
+ ),
285
+ Effect.tap((stop) =>
286
+ Effect.sync(() => {
287
+ if (stop) token.active = false;
288
+ }),
289
+ ),
290
+ Effect.ensuring(Deferred.succeed(processed, undefined)),
291
+ ),
292
+ ),
293
+ Stream.takeUntil(Boolean),
294
+ Stream.runDrain,
295
+ Effect.ensuring(
296
+ Effect.sync(() => {
297
+ token.active = false;
298
+ }),
299
+ ),
300
+ );
301
+ });
302
+ return { barrier, consume };
303
+});
apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts
+6
-2
@@ -546,8 +546,12 @@ layer("ProviderEventIngestorV2", (it) => {
546
event: { type: "node.updated", driver: CODEX_DRIVER, node: sent },
547
});
548
549
- assert.lengthOf(yield* ingestNode(node), 1);
550
- assert.isEmpty(yield* ingestNode({ ...node }));
549
+ yield* eventSink.withProviderBatch(
550
+ Effect.gen(function* () {
551
+ assert.lengthOf(yield* ingestNode(node), 1);
552
+ assert.isEmpty(yield* ingestNode({ ...node }));
553
+ }),
554
+ );
555
yield* eventSink.write({
556
events: [
557
{
apps/server/src/orchestration-v2/RunExecutionService.test.ts
+21
-10
@@ -55,6 +55,17 @@ import * as RunExecutionService from "./RunExecutionService.ts";
55
import * as RunFinalizationService from "./RunFinalizationService.ts";
56
import * as TurnItemPositionStore from "./TurnItemPositionStore.ts";
57
58
+const mockEventSink = (implementation: Partial<EventSink.EventSinkV2["Service"]>) =>
59
+ Layer.mock(EventSink.EventSinkV2)({
60
+ consumeProviderEvents: (source, process, after) =>
61
+ source.pipe(
62
+ Stream.mapEffect((event) => process(event).pipe(Effect.andThen(after))),
63
+ Stream.takeUntil(Boolean),
64
+ Stream.runDrain,
65
+ ),
66
+ ...implementation,
67
+ });
68
+
69
const driver = ProviderDriverKind.make("codex");
70
const rootAppThread = (id: ThreadId) =>
71
({
@@ -73,7 +84,7 @@ const layerRunExecutionTest = RunExecutionService.layer.pipe(
84
PreviewAgentActivity.layer,
85
Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }),
86
layerTurnItemPositionsTest,
76
- Layer.mock(EventSink.EventSinkV2)({}),
87
+ mockEventSink({}),
88
IdAllocator.layer,
89
Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
90
ingestNormalized: () => Effect.succeed([]),
@@ -900,7 +911,7 @@ it.effect("passes the thread's MCP app context to the provider under a safe key"
911
captureBaseline: () => Effect.void,
912
}),
913
layerTurnItemPositionsTest,
903
- Layer.mock(EventSink.EventSinkV2)({}),
914
+ mockEventSink({}),
915
IdAllocator.layer,
916
Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
917
ingestNormalized: () => Effect.succeed([]),
@@ -929,7 +940,7 @@ it.effect("fails the run when its ownership check cannot be read before calling
940
PreviewAgentActivity.layer,
941
Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }),
942
layerTurnItemPositionsTest,
932
- Layer.mock(EventSink.EventSinkV2)({
943
+ mockEventSink({
944
writeIfRunCurrent: (input) =>
945
Ref.update(writes, (current) => [...current, ...input.events]).pipe(
946
Effect.as({ committed: true, storedEvents: [] }),
@@ -1140,7 +1151,7 @@ it.effect("starts the provider when checkpoint baseline capture fails", () =>
1151
),
1152
}),
1153
layerTurnItemPositionsTest,
1143
- Layer.mock(EventSink.EventSinkV2)({
1154
+ mockEventSink({
1155
writeWithEffects: (input) =>
1156
Ref.update(writes, (current) => [
1157
...current,
@@ -1254,7 +1265,7 @@ it.effect.each(["failure", "interruption", "stale-attempt", "start-guard"] as co
1265
scenario === "start-guard" ? Effect.void : Effect.die("not reached"),
1266
}),
1267
layerTurnItemPositionsTest,
1257
- Layer.mock(EventSink.EventSinkV2)({
1268
+ mockEventSink({
1269
writeIfRunCurrent: (input) =>
1270
Effect.gen(function* () {
1271
assert.equal(input.threadId, threadId);
@@ -1434,7 +1445,7 @@ it.effect("ingests new member threads of an owned child after the root turn term
1445
PreviewAgentActivity.layer,
1446
Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }),
1447
layerTurnItemPositionsTest,
1437
- Layer.mock(EventSink.EventSinkV2)({
1448
+ mockEventSink({
1449
write: () => Effect.succeed([]),
1450
writeWithEffects: (input) =>
1451
Effect.gen(function* () {
@@ -1694,7 +1705,7 @@ it.effect("ends a nested native subagent and its thread when the root run is int
1705
PreviewAgentActivity.layer,
1706
Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }),
1707
layerTurnItemPositionsTest,
1697
- Layer.mock(EventSink.EventSinkV2)({
1708
+ mockEventSink({
1709
write: () => Effect.succeed([]),
1710
writeWithEffects: (input) =>
1711
Effect.as(
@@ -2107,7 +2118,7 @@ it.effect(
2118
captureBaseline: () => Effect.void,
2119
}),
2120
layerTurnItemPositionsTest,
2110
- Layer.mock(EventSink.EventSinkV2)({
2121
+ mockEventSink({
2122
write: () => Effect.succeed([]),
2123
writeWithEffects: (input) =>
2124
Effect.gen(function* () {
@@ -3193,7 +3204,7 @@ function captureRootRunTermination(input: {
3204
}),
3205
Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }),
3206
layerTurnItemPositionsTest,
3196
- Layer.mock(EventSink.EventSinkV2)({
3207
+ mockEventSink({
3208
write: (payload) =>
3209
Effect.gen(function* () {
3210
for (const event of payload.events) {
@@ -3715,7 +3726,7 @@ function runBackgroundItemScenario(
3726
Layer.succeed(PreviewAgentActivity.PreviewAgentActivity, previewActivity),
3727
Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }),
3728
layerTurnItemPositionsTest,
3718
- Layer.mock(EventSink.EventSinkV2)({
3729
+ mockEventSink({
3730
write: () => Effect.succeed([]),
3731
writeWithEffects: (input) =>
3732
Effect.gen(function* () {
apps/server/src/orchestration-v2/RunExecutionService.ts
+335
-336
@@ -35,7 +35,6 @@ import * as Layer from "effect/Layer";
35
import * as Ref from "effect/Ref";
36
import * as Schedule from "effect/Schedule";
37
import * as Schema from "effect/Schema";
38
-import * as Stream from "effect/Stream";
38
39
import * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts";
40
import * as McpAppModelContext from "../mcpApps/McpAppModelContext.ts";
@@ -1322,350 +1321,350 @@ export const layer: Layer.Layer<
1321
// Ends once the root turn and the children it drives settle. Background commands such
1322
// as a dev server keep this consumer alive afterwards and must not pin preview focus.
1323
const endPreviewActivity = yield* previewAgentActivity.begin(input.run.threadId);
1325
- const providerEventFiber = yield* eventSubscription.events.pipe(
1326
- Stream.mapEffect((event) =>
1327
- Effect.gen(function* () {
1328
- const previous = yield* Ref.get(eventRouting);
1329
- const [accepted, routing] = routeProviderEvent(event, routeIdentity, previous);
1330
- yield* Ref.set(eventRouting, routing);
1331
- if (
1332
- event.type === "subagent.updated" &&
1333
- event.subagent.childThreadId !== null &&
1334
- previous.ownedThreadIds.has(event.subagent.childThreadId) &&
1335
- !routing.ownedThreadIds.has(event.subagent.childThreadId)
1336
- ) {
1337
- const moved = event.subagent;
1338
- const detached = yield* Ref.modify(openRunOwnedSubagents, (current) => {
1339
- const subagentIds = new Set([moved.id]);
1340
- for (const subagent of current.subagents.values()) {
1341
- if (subagent.threadId === moved.childThreadId) subagentIds.add(subagent.id);
1342
- }
1343
- const childTurnItems = new Map(
1344
- [...current.childTurnItems].filter(
1345
- ([, item]) => item.threadId !== moved.childThreadId,
1346
- ),
1347
- );
1348
- const turnItems = new Map(current.turnItems);
1349
- turnItems.delete(moved.id);
1350
- const detachedItems = new Set(
1351
- [...current.childTurnItems.values(), ...current.turnItems.values()]
1352
- .filter(
1353
- (item) => item.threadId === moved.childThreadId || item.nodeId === moved.id,
1354
- )
1355
- .map((item) => item.id),
1356
- );
1357
- return [
1358
- { itemIds: detachedItems, subagentIds },
1359
- {
1360
- subagents: new Map(
1361
- [...current.subagents].filter(([id]) => !subagentIds.has(id)),
1362
- ),
1363
- turnItems,
1364
- childTurnItems,
1365
- nodes: new Map(
1366
- [...current.nodes].filter(
1367
- ([id, node]) => id !== moved.id && node.threadId !== moved.childThreadId,
1368
- ),
1369
- ),
1370
- linkedChildThreadIds: new Set(
1371
- [...current.linkedChildThreadIds].filter(
1372
- (id) => id !== moved.childThreadId,
1373
- ),
1374
- ),
1375
- },
1376
- ];
1377
- });
1378
- yield* Ref.update(
1379
- activeChildSubagents,
1380
- (current) =>
1381
- new Map(
1382
- [...current].filter(
1383
- ([id, threadId]) =>
1384
- !detached.subagentIds.has(id) && threadId !== moved.childThreadId,
1385
- ),
1386
- ),
1387
- );
1388
- yield* Ref.update(
1389
- activeChildProviderTurns,
1390
- (current) =>
1391
- new Set([...current].filter((id) => routing.ownedProviderTurns.has(id))),
1392
- );
1393
- for (const id of pendingBackgroundProviderTurns) {
1394
- if (!routing.ownedProviderTurns.has(id)) {
1395
- pendingBackgroundProviderTurns.delete(id);
1396
- }
1397
- }
1398
- yield* Ref.update(
1399
- activeBackgroundTurnItems,
1400
- (current) => new Set([...current].filter((id) => !detached.itemIds.has(id))),
1401
- );
1402
- }
1403
- return { event, accepted };
1404
- }),
1405
- ),
1406
- Stream.tap(({ event, accepted }) =>
1407
- Effect.gen(function* () {
1408
- if (!accepted) return;
1409
- if (
1410
- event.type === "provider_turn.updated" &&
1411
- event.providerTurn.runAttemptId === input.attempt.id &&
1412
- event.providerTurn.providerThreadId === input.providerThread.id
1413
- ) {
1414
- yield* Ref.set(latestRootProviderTurn, event.providerTurn);
1324
+ // Per-event routing and lifecycle bookkeeping share the run's tracing boundary.
1325
+ const routeEvent = Effect.fnUntraced(function* (
1326
+ event: ProviderAdapter.ProviderAdapterV2Event,
1327
+ ) {
1328
+ const previous = yield* Ref.get(eventRouting);
1329
+ const [accepted, routing] = routeProviderEvent(event, routeIdentity, previous);
1330
+ yield* Ref.set(eventRouting, routing);
1331
+ if (
1332
+ event.type === "subagent.updated" &&
1333
+ event.subagent.childThreadId !== null &&
1334
+ previous.ownedThreadIds.has(event.subagent.childThreadId) &&
1335
+ !routing.ownedThreadIds.has(event.subagent.childThreadId)
1336
+ ) {
1337
+ const moved = event.subagent;
1338
+ const detached = yield* Ref.modify(openRunOwnedSubagents, (current) => {
1339
+ const subagentIds = new Set([moved.id]);
1340
+ for (const subagent of current.subagents.values()) {
1341
+ if (subagent.threadId === moved.childThreadId) subagentIds.add(subagent.id);
1342
}
1416
- let storedEventCount = 0;
1417
- const deliveredEvent = filterAssistantEvent(
1418
- event,
1419
- DateTime.toEpochMillis(yield* DateTime.now),
1343
+ const childTurnItems = new Map(
1344
+ [...current.childTurnItems].filter(
1345
+ ([, item]) => item.threadId !== moved.childThreadId,
1346
+ ),
1347
);
1421
- if (deliveredEvent) {
1422
- const rootTerminalAlreadySeen = yield* Ref.get(rootTerminalSeen);
1423
- const isRootProviderThreadUpdate =
1424
- event.type === "provider_thread.updated" &&
1425
- event.providerThread.id === input.providerThread.id;
1426
- const nativeChildTurn =
1427
- event.type === "provider_turn.updated" &&
1428
- event.providerTurn.providerThreadId !== input.providerThread.id;
1429
- if (event.type === "provider_turn.updated" && !nativeChildTurn) {
1430
- firstRootProviderTurnId ??= event.providerTurn.id;
1431
- }
1432
- const storedEvents = yield* providerEventIngestor.ingestNormalized({
1433
- analyticsContext: nativeChildTurn
1434
- ? {
1435
- ...analyticsContext,
1436
- origin: NATIVE_SUBAGENT_TURN_ORIGIN,
1437
- segment: "native_child",
1438
- }
1439
- : {
1440
- ...analyticsContext,
1441
- segment:
1442
- event.type === "provider_turn.updated" &&
1443
- event.providerTurn.id !== firstRootProviderTurnId
1444
- ? "continuation"
1445
- : input.attempt.reason,
1446
- },
1447
- providerSessionId: input.providerSessionId,
1448
- providerInstanceId: input.run.providerInstanceId,
1449
- threadId: input.run.threadId,
1450
- runId: input.run.id,
1451
- nodeId: input.rootNode.id,
1452
- event: deliveredEvent,
1453
- ...(isRootProviderThreadUpdate
1454
- ? rootTerminalAlreadySeen
1455
- ? {
1456
- writeIfProviderThreadOwner: {
1457
- providerThreadId: input.providerThread.id,
1458
- runId: input.run.id,
1459
- activeAttemptId: input.attempt.id,
1460
- expectedLastRunOrdinal: input.run.ordinal,
1461
- },
1462
- }
1463
- : {
1464
- writeIfRunCurrent: {
1465
- runId: input.run.id,
1466
- activeAttemptId: input.attempt.id,
1467
- expectedStatus: "running" as const,
1468
- },
1469
- }
1470
- : {}),
1471
- });
1472
- storedEventCount = storedEvents.length;
1473
- if (
1474
- isRootProviderThreadUpdate &&
1475
- rootTerminalAlreadySeen &&
1476
- storedEventCount === 0
1477
- ) {
1478
- yield* Ref.set(providerThreadOwnerLost, true);
1479
- }
1480
- } else if (
1481
- event.type === "turn_item.updated" &&
1482
- (event.turnItem.type === "assistant_message" ||
1483
- event.turnItem.type === "reasoning") &&
1484
- event.turnItem.streaming &&
1485
- !reservedHeldTurnItemIds.has(event.turnItem.id)
1486
- ) {
1487
- yield* turnItemPositions.allocate({
1488
- threadId: event.turnItem.threadId,
1489
- turnItemId: event.turnItem.id,
1490
- runId: event.turnItem.runId,
1491
- });
1492
- reservedHeldTurnItemIds.add(event.turnItem.id);
1493
- }
1494
- if (event.type === "provider_thread.updated") {
1495
- if (event.providerThread.id === input.providerThread.id && storedEventCount > 0) {
1496
- yield* Ref.set(latestProviderThread, event.providerThread);
1497
- }
1498
- }
1499
- if (
1500
- event.type === "turn_item.updated" &&
1501
- event.turnItem.providerTurnId === (yield* Ref.get(eventRouting)).rootProviderTurnId
1502
- ) {
1503
- yield* Ref.update(latestTurnItemOrdinal, (current) =>
1504
- Math.max(current, event.turnItem.ordinal),
1505
- );
1506
- }
1507
- if (event.type === "turn.terminal") {
1508
- yield* Ref.set(terminalEvent, event);
1509
- yield* Ref.set(rootTerminalSeen, true);
1510
- yield* finalizeRootRun(event);
1511
- }
1512
- if (event.type === "background_work.updated") {
1513
- if (event.pending) {
1514
- pendingBackgroundProviderTurns.add(event.providerTurnId);
1515
- } else {
1516
- pendingBackgroundProviderTurns.delete(event.providerTurnId);
1517
- }
1348
+ const turnItems = new Map(current.turnItems);
1349
+ turnItems.delete(moved.id);
1350
+ const detachedItems = new Set(
1351
+ [...current.childTurnItems.values(), ...current.turnItems.values()]
1352
+ .filter(
1353
+ (item) => item.threadId === moved.childThreadId || item.nodeId === moved.id,
1354
+ )
1355
+ .map((item) => item.id),
1356
+ );
1357
+ return [
1358
+ { itemIds: detachedItems, subagentIds },
1359
+ {
1360
+ subagents: new Map([...current.subagents].filter(([id]) => !subagentIds.has(id))),
1361
+ turnItems,
1362
+ childTurnItems,
1363
+ nodes: new Map(
1364
+ [...current.nodes].filter(
1365
+ ([id, node]) => id !== moved.id && node.threadId !== moved.childThreadId,
1366
+ ),
1367
+ ),
1368
+ linkedChildThreadIds: new Set(
1369
+ [...current.linkedChildThreadIds].filter((id) => id !== moved.childThreadId),
1370
+ ),
1371
+ },
1372
+ ];
1373
+ });
1374
+ yield* Ref.update(
1375
+ activeChildSubagents,
1376
+ (current) =>
1377
+ new Map(
1378
+ [...current].filter(
1379
+ ([id, threadId]) =>
1380
+ !detached.subagentIds.has(id) && threadId !== moved.childThreadId,
1381
+ ),
1382
+ ),
1383
+ );
1384
+ yield* Ref.update(
1385
+ activeChildProviderTurns,
1386
+ (current) => new Set([...current].filter((id) => routing.ownedProviderTurns.has(id))),
1387
+ );
1388
+ for (const id of pendingBackgroundProviderTurns) {
1389
+ if (!routing.ownedProviderTurns.has(id)) {
1390
+ pendingBackgroundProviderTurns.delete(id);
1391
}
1519
- yield* trackChildLifecycle(event, deliveredEvent !== null);
1520
- }),
1521
- ),
1522
- // Also after events this run does not accept, such as a child handed to another run.
1523
- Stream.tap(() =>
1392
+ }
1393
+ yield* Ref.update(
1394
+ activeBackgroundTurnItems,
1395
+ (current) => new Set([...current].filter((id) => !detached.itemIds.has(id))),
1396
+ );
1397
+ }
1398
+ return { event, accepted };
1399
+ });
1400
+ const ingestEvent = Effect.fnUntraced(function* ({
1401
+ event,
1402
+ accepted,
1403
+ }: {
1404
+ event: ProviderAdapter.ProviderAdapterV2Event;
1405
+ accepted: boolean;
1406
+ }) {
1407
+ if (!accepted) return;
1408
+ if (
1409
+ event.type === "provider_turn.updated" &&
1410
+ event.providerTurn.runAttemptId === input.attempt.id &&
1411
+ event.providerTurn.providerThreadId === input.providerThread.id
1412
+ ) {
1413
+ yield* Ref.set(latestRootProviderTurn, event.providerTurn);
1414
+ }
1415
+ let storedEventCount = 0;
1416
+ const deliveredEvent = filterAssistantEvent(
1417
+ event,
1418
+ DateTime.toEpochMillis(yield* DateTime.now),
1419
+ );
1420
+ if (deliveredEvent) {
1421
+ const rootTerminalAlreadySeen = yield* Ref.get(rootTerminalSeen);
1422
+ const isRootProviderThreadUpdate =
1423
+ event.type === "provider_thread.updated" &&
1424
+ event.providerThread.id === input.providerThread.id;
1425
+ const nativeChildTurn =
1426
+ event.type === "provider_turn.updated" &&
1427
+ event.providerTurn.providerThreadId !== input.providerThread.id;
1428
+ if (event.type === "provider_turn.updated" && !nativeChildTurn) {
1429
+ firstRootProviderTurnId ??= event.providerTurn.id;
1430
+ }
1431
+ const storedEvents = yield* providerEventIngestor.ingestNormalized({
1432
+ analyticsContext: nativeChildTurn
1433
+ ? {
1434
+ ...analyticsContext,
1435
+ origin: NATIVE_SUBAGENT_TURN_ORIGIN,
1436
+ segment: "native_child",
1437
+ }
1438
+ : {
1439
+ ...analyticsContext,
1440
+ segment:
1441
+ event.type === "provider_turn.updated" &&
1442
+ event.providerTurn.id !== firstRootProviderTurnId
1443
+ ? "continuation"
1444
+ : input.attempt.reason,
1445
+ },
1446
+ providerSessionId: input.providerSessionId,
1447
+ providerInstanceId: input.run.providerInstanceId,
1448
+ threadId: input.run.threadId,
1449
+ runId: input.run.id,
1450
+ nodeId: input.rootNode.id,
1451
+ event: deliveredEvent,
1452
+ ...(isRootProviderThreadUpdate
1453
+ ? rootTerminalAlreadySeen
1454
+ ? {
1455
+ writeIfProviderThreadOwner: {
1456
+ providerThreadId: input.providerThread.id,
1457
+ runId: input.run.id,
1458
+ activeAttemptId: input.attempt.id,
1459
+ expectedLastRunOrdinal: input.run.ordinal,
1460
+ },
1461
+ }
1462
+ : {
1463
+ writeIfRunCurrent: {
1464
+ runId: input.run.id,
1465
+ activeAttemptId: input.attempt.id,
1466
+ expectedStatus: "running" as const,
1467
+ },
1468
+ }
1469
+ : {}),
1470
+ });
1471
+ storedEventCount = storedEvents.length;
1472
+ if (isRootProviderThreadUpdate && rootTerminalAlreadySeen && storedEventCount === 0) {
1473
+ yield* Ref.set(providerThreadOwnerLost, true);
1474
+ }
1475
+ } else if (
1476
+ event.type === "turn_item.updated" &&
1477
+ (event.turnItem.type === "assistant_message" || event.turnItem.type === "reasoning") &&
1478
+ event.turnItem.streaming &&
1479
+ !reservedHeldTurnItemIds.has(event.turnItem.id)
1480
+ ) {
1481
+ yield* turnItemPositions.allocate({
1482
+ threadId: event.turnItem.threadId,
1483
+ turnItemId: event.turnItem.id,
1484
+ runId: event.turnItem.runId,
1485
+ });
1486
+ reservedHeldTurnItemIds.add(event.turnItem.id);
1487
+ }
1488
+ if (event.type === "provider_thread.updated") {
1489
+ if (event.providerThread.id === input.providerThread.id && storedEventCount > 0) {
1490
+ yield* Ref.set(latestProviderThread, event.providerThread);
1491
+ }
1492
+ }
1493
+ if (
1494
+ event.type === "turn_item.updated" &&
1495
+ event.turnItem.providerTurnId === (yield* Ref.get(eventRouting)).rootProviderTurnId
1496
+ ) {
1497
+ yield* Ref.update(latestTurnItemOrdinal, (current) =>
1498
+ Math.max(current, event.turnItem.ordinal),
1499
+ );
1500
+ }
1501
+ if (event.type === "turn.terminal") {
1502
+ yield* Ref.set(terminalEvent, event);
1503
+ yield* Ref.set(rootTerminalSeen, true);
1504
+ yield* finalizeRootRun(event);
1505
+ }
1506
+ if (event.type === "background_work.updated") {
1507
+ if (event.pending) {
1508
+ pendingBackgroundProviderTurns.add(event.providerTurnId);
1509
+ } else {
1510
+ pendingBackgroundProviderTurns.delete(event.providerTurnId);
1511
+ }
1512
+ }
1513
+ yield* trackChildLifecycle(event, deliveredEvent !== null);
1514
+ });
1515
+ const providerEventFiber = yield* eventSink
1516
+ .consumeProviderEvents(
1517
+ eventSubscription.events,
1518
+ (event) =>
1519
+ Effect.gen(function* () {
1520
+ yield* ingestEvent(yield* routeEvent(event));
1521
+ }),
1522
Effect.gen(function* () {
1523
if (yield* agentTurnsSettled) yield* endPreviewActivity;
1524
+ return yield* shouldStopProviderEventIngestion;
1525
}),
1527
- ),
1528
- Stream.takeUntilEffect(() => shouldStopProviderEventIngestion),
1529
- Stream.runDrain,
1530
- Effect.mapError((cause) => new RunExecutionIngestError({ runId: input.run.id, cause })),
1531
- Effect.flatMap(() =>
1532
- Effect.gen(function* () {
1533
- const terminal = yield* Ref.get(terminalEvent);
1534
- if (terminal === null) {
1535
- // A session release interrupts or fails the stream; a clean
1536
- // end without a terminal means the run would never settle.
1537
- return yield* new RunExecutionIngestError({
1538
- runId: input.run.id,
1539
- cause: new ProviderAdapter.ProviderAdapterEventStreamError({
1540
- driver: input.providerThread.driver,
1541
- providerSessionId: input.providerSessionId,
1542
- cause: "Provider event stream ended without a root terminal.",
1543
- }),
1544
- });
1545
- }
1546
- yield* finalizeRootRun(terminal);
1547
- }),
1548
- ),
1549
- // Server shutdown interrupts the subscription and leaves the run to
1550
- // restart recovery; every other failure fails the run.
1551
- Effect.catchCauseIf(
1552
- (cause) => !Cause.hasInterruptsOnly(cause),
1553
- (cause) =>
1526
+ Ref.get(rootTerminalSeen).pipe(Effect.map((seen) => !seen)),
1527
+ )
1528
+ .pipe(
1529
+ Effect.mapError((cause) => new RunExecutionIngestError({ runId: input.run.id, cause })),
1530
+ Effect.flatMap(() =>
1531
Effect.gen(function* () {
1555
- const completedAt = yield* DateTime.now;
1556
- yield* Effect.logWarning("orchestration V2 provider event ingestion failed", {
1557
- runId: input.run.id,
1558
- cause,
1559
- });
1560
- if (yield* Ref.get(rootRunFinalized)) return;
1561
- const providerThread = yield* Ref.get(latestProviderThread);
1562
- const routing = yield* Ref.get(eventRouting);
1563
- const rootProviderTurn = yield* Ref.get(latestRootProviderTurn);
1564
- // Stop background work too; the failed consumer cannot persist terminal events.
1565
- const abandonedProviderTurn =
1566
- routing.rootProviderTurnId !== null && !routing.rootTurnEnded
1567
- ? yield* input.session
1568
- .interruptTurn({
1569
- providerThread,
1570
- providerTurnId: routing.rootProviderTurnId,
1571
- requestRuntimeRestart: true,
1572
- })
1573
- .pipe(
1574
- Effect.as(true),
1575
- Effect.catchCauseIf(
1576
- (interruptCause) => !Cause.hasInterruptsOnly(interruptCause),
1577
- (interruptCause) =>
1578
- Effect.logWarning(
1579
- "orchestration V2 failed to interrupt turn after ingestion failure",
1580
- { runId: input.run.id, cause: interruptCause },
1581
- ).pipe(Effect.as(false)),
1582
- ),
1583
- // A failed interrupt leaves the stored turn running so the next start
1584
- // retries it. The snapshot is cached before it is published, so it may
1585
- // not be stored: replay a terminal one as-is.
1586
- Effect.map((interrupted) =>
1587
- !interrupted || rootProviderTurn === null
1588
- ? null
1589
- : isTerminalProviderTurnStatus(rootProviderTurn.status)
1590
- ? rootProviderTurn
1591
- : {
1592
- ...rootProviderTurn,
1593
- status: "interrupted" as const,
1594
- completedAt,
1595
- },
1596
- ),
1597
- )
1598
- : null;
1599
- const latestItemOrdinal = yield* Ref.get(latestTurnItemOrdinal);
1600
- const openSubagents = yield* Ref.get(openRunOwnedSubagents);
1601
- const squashed = Cause.squash(cause);
1602
- let nextRetryWarningAt = completedAt;
1603
- yield* writeFinalRunEvents({
1604
- run: input.run,
1605
- rootNode: input.rootNode,
1606
- checkpointScope: input.checkpointScope,
1607
- providerThread,
1608
- attempt: input.attempt,
1609
- writeIfRunCurrent: {
1610
- activeAttemptId: input.attempt.id,
1611
- expectedStatus: "running",
1612
- },
1613
- openRunOwnedSubagents: openSubagents,
1614
- ...(abandonedProviderTurn === null ? {} : { abandonedProviderTurn }),
1615
- completedAt,
1616
- terminal: makeFailedTerminalEvent(
1617
- makeProviderFailure({
1618
- cause: squashed,
1619
- message:
1620
- knownProviderFailureMessage(squashed) ??
1621
- userFacingDatabaseErrorMessage(squashed),
1622
- class: "unknown",
1623
- }),
1624
- latestItemOrdinal + 1,
1625
- ),
1626
- failureItemPersisted: false,
1627
- refreshAfterTurn,
1628
- }).pipe(
1629
- // Storage failures can be defects; retry both failure channels.
1630
- Effect.catchCause((writeCause) =>
1631
- Effect.gen(function* () {
1632
- if (Cause.hasInterruptsOnly(writeCause)) {
1633
- return yield* Effect.failCause(writeCause);
1634
- }
1635
- const now = yield* DateTime.now;
1636
- if (DateTime.isGreaterThanOrEqualTo(now, nextRetryWarningAt)) {
1637
- yield* Effect.logWarning(
1638
- "orchestration V2 run failure finalization retry",
1639
- {
1640
- runId: input.run.id,
1641
- attemptId: input.attempt.id,
1642
- cause: writeCause,
1643
- },
1644
- );
1645
- // Persistent storage failures must not flood the same disk.
1646
- nextRetryWarningAt = DateTime.add(now, { minutes: 10 });
1647
- }
1648
- return yield* Effect.fail(writeCause);
1532
+ const terminal = yield* Ref.get(terminalEvent);
1533
+ if (terminal === null) {
1534
+ // A session release interrupts or fails the stream; a clean
1535
+ // end without a terminal means the run would never settle.
1536
+ return yield* new RunExecutionIngestError({
1537
+ runId: input.run.id,
1538
+ cause: new ProviderAdapter.ProviderAdapterEventStreamError({
1539
+ driver: input.providerThread.driver,
1540
+ providerSessionId: input.providerSessionId,
1541
+ cause: "Provider event stream ended without a root terminal.",
1542
}),
1650
- ),
1651
- Effect.retry({
1652
- schedule: Schedule.exponential("1 second").pipe(
1653
- Schedule.modifyDelay(({ duration }) =>
1654
- Effect.succeed(Duration.min(duration, Duration.seconds(30))),
1655
- ),
1656
- ),
1657
- }),
1658
- // Startup may cancel the consumer and fail its own terminal
1659
- // write. Only service shutdown or the atomic gate ends recovery.
1660
- Effect.forkIn(scope),
1661
- Effect.flatMap(Fiber.join),
1662
- );
1543
+ });
1544
+ }
1545
+ yield* finalizeRootRun(terminal);
1546
}),
1664
- ),
1665
- Effect.ensuring(endPreviewActivity),
1666
- Effect.ensuring(eventSubscription.close),
1667
- Effect.forkDetach,
1668
- );
1547
+ ),
1548
+ // Server shutdown interrupts the subscription and leaves the run to
1549
+ // restart recovery; every other failure fails the run.
1550
+ Effect.catchCauseIf(
1551
+ (cause) => !Cause.hasInterruptsOnly(cause),
1552
+ (cause) =>
1553
+ Effect.gen(function* () {
1554
+ const completedAt = yield* DateTime.now;
1555
+ yield* Effect.logWarning("orchestration V2 provider event ingestion failed", {
1556
+ runId: input.run.id,
1557
+ cause,
1558
+ });
1559
+ if (yield* Ref.get(rootRunFinalized)) return;
1560
+ const providerThread = yield* Ref.get(latestProviderThread);
1561
+ const routing = yield* Ref.get(eventRouting);
1562
+ const rootProviderTurn = yield* Ref.get(latestRootProviderTurn);
1563
+ // Stop background work too; the failed consumer cannot persist terminal events.
1564
+ const abandonedProviderTurn =
1565
+ routing.rootProviderTurnId !== null && !routing.rootTurnEnded
1566
+ ? yield* input.session
1567
+ .interruptTurn({
1568
+ providerThread,
1569
+ providerTurnId: routing.rootProviderTurnId,
1570
+ requestRuntimeRestart: true,
1571
+ })
1572
+ .pipe(
1573
+ Effect.as(true),
1574
+ Effect.catchCauseIf(
1575
+ (interruptCause) => !Cause.hasInterruptsOnly(interruptCause),
1576
+ (interruptCause) =>
1577
+ Effect.logWarning(
1578
+ "orchestration V2 failed to interrupt turn after ingestion failure",
1579
+ { runId: input.run.id, cause: interruptCause },
1580
+ ).pipe(Effect.as(false)),
1581
+ ),
1582
+ // A failed interrupt leaves the stored turn running so the next start
1583
+ // retries it. The snapshot is cached before it is published, so it may
1584
+ // not be stored: replay a terminal one as-is.
1585
+ Effect.map((interrupted) =>
1586
+ !interrupted || rootProviderTurn === null
1587
+ ? null
1588
+ : isTerminalProviderTurnStatus(rootProviderTurn.status)
1589
+ ? rootProviderTurn
1590
+ : {
1591
+ ...rootProviderTurn,
1592
+ status: "interrupted" as const,
1593
+ completedAt,
1594
+ },
1595
+ ),
1596
+ )
1597
+ : null;
1598
+ const latestItemOrdinal = yield* Ref.get(latestTurnItemOrdinal);
1599
+ const openSubagents = yield* Ref.get(openRunOwnedSubagents);
1600
+ const squashed = Cause.squash(cause);
1601
+ let nextRetryWarningAt = completedAt;
1602
+ yield* writeFinalRunEvents({
1603
+ run: input.run,
1604
+ rootNode: input.rootNode,
1605
+ checkpointScope: input.checkpointScope,
1606
+ providerThread,
1607
+ attempt: input.attempt,
1608
+ writeIfRunCurrent: {
1609
+ activeAttemptId: input.attempt.id,
1610
+ expectedStatus: "running",
1611
+ },
1612
+ openRunOwnedSubagents: openSubagents,
1613
+ ...(abandonedProviderTurn === null ? {} : { abandonedProviderTurn }),
1614
+ completedAt,
1615
+ terminal: makeFailedTerminalEvent(
1616
+ makeProviderFailure({
1617
+ cause: squashed,
1618
+ message:
1619
+ knownProviderFailureMessage(squashed) ??
1620
+ userFacingDatabaseErrorMessage(squashed),
1621
+ class: "unknown",
1622
+ }),
1623
+ latestItemOrdinal + 1,
1624
+ ),
1625
+ failureItemPersisted: false,
1626
+ refreshAfterTurn,
1627
+ }).pipe(
1628
+ // Storage failures can be defects; retry both failure channels.
1629
+ Effect.catchCause((writeCause) =>
1630
+ Effect.gen(function* () {
1631
+ if (Cause.hasInterruptsOnly(writeCause)) {
1632
+ return yield* Effect.failCause(writeCause);
1633
+ }
1634
+ const now = yield* DateTime.now;
1635
+ if (DateTime.isGreaterThanOrEqualTo(now, nextRetryWarningAt)) {
1636
+ yield* Effect.logWarning(
1637
+ "orchestration V2 run failure finalization retry",
1638
+ {
1639
+ runId: input.run.id,
1640
+ attemptId: input.attempt.id,
1641
+ cause: writeCause,
1642
+ },
1643
+ );
1644
+ // Persistent storage failures must not flood the same disk.
1645
+ nextRetryWarningAt = DateTime.add(now, { minutes: 10 });
1646
+ }
1647
+ return yield* Effect.fail(writeCause);
1648
+ }),
1649
+ ),
1650
+ Effect.retry({
1651
+ schedule: Schedule.exponential("1 second").pipe(
1652
+ Schedule.modifyDelay(({ duration }) =>
1653
+ Effect.succeed(Duration.min(duration, Duration.seconds(30))),
1654
+ ),
1655
+ ),
1656
+ }),
1657
+ // Startup may cancel the consumer and fail its own terminal
1658
+ // write. Only service shutdown or the atomic gate ends recovery.
1659
+ Effect.forkIn(scope),
1660
+ Effect.flatMap(Fiber.join),
1661
+ );
1662
+ }),
1663
+ ),
1664
+ Effect.ensuring(endPreviewActivity),
1665
+ Effect.ensuring(eventSubscription.close),
1666
+ Effect.forkDetach,
1667
+ );
1668
1669
// A provider turn is a sign that the session is still alive. Keep
1670
// its already-issued MCP credential valid even when the agent goes
apps/server/src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts
+43
-2
@@ -1,9 +1,14 @@
1
-import { describe } from "@effect/vitest";
1
+import { assert, describe, it } from "@effect/vitest";
2
+import * as Tracer from "effect/Tracer";
3
import * as Effect from "effect/Effect";
4
5
import { materializeReplayTranscriptWorkspace } from "@t3tools/provider-testing/replayTranscript";
6
import { CodexOrchestratorReplayHarness } from "../Adapters/CodexAdapterV2.testkit.ts";
6
-import { itRunsReplayFixtures, runReplayFixture } from "./ProviderReplayFixtures.ts";
7
+import {
8
+ findReplayFixture,
9
+ itRunsReplayFixtures,
10
+ runReplayFixture,
11
+} from "./ProviderReplayFixtures.ts";
12
13
describe("Codex replay fixtures", () => {
14
itRunsReplayFixtures("codex", (input) =>
@@ -15,3 +20,39 @@ describe("Codex replay fixtures", () => {
20
}),
21
);
22
});
23
+
24
+it.effect(
25
+ "persists recorded Codex snapshots through provider transactions in the real run consumer",
26
+ () =>
27
+ Effect.gen(function* () {
28
+ const fixture = findReplayFixture("multi_turn")!;
29
+ const driver = fixture.providers.find((provider) => provider.driver === "codex")!;
30
+ const writes: number[] = [];
31
+ const tracer = Tracer.make({
32
+ span(options) {
33
+ const span = new Tracer.NativeSpan(options);
34
+ const end = span.end.bind(span);
35
+ span.end = (time, exit) => {
36
+ end(time, exit);
37
+ if (span.name === "orchestrationV2.EventSink.providerBatch") {
38
+ writes.push(Number(span.attributes.get("orchestration_v2.batch_writes")));
39
+ }
40
+ };
41
+ return span;
42
+ },
43
+ });
44
+ yield* runReplayFixture({
45
+ fixtureName: fixture.name,
46
+ buildInput: fixture.buildInput,
47
+ driver,
48
+ harness: CodexOrchestratorReplayHarness,
49
+ materializeWorkspace: (transcript, workspace) =>
50
+ Effect.succeed(materializeReplayTranscriptWorkspace(transcript, workspace)),
51
+ }).pipe(Effect.withTracer(tracer), Effect.withTracerEnabled(true));
52
+ assert.equal(
53
+ writes.reduce((total, count) => total + count, 0),
54
+ 6,
55
+ );
56
+ assert.isTrue(writes.every((count) => count <= 3));
57
+ }),
58
+);
packages/provider-opencode/src/server/adapter.test.ts
+4
@@ -712,6 +712,10 @@ describe("OpenCodeAdapterV2", () => {
712
}),
713
);
714
const events = yield* Fiber.join(received);
715
+ assert.deepEqual(
716
+ events.map((event) => event.type),
717
+ ["turn_item.updated", "runtime_request.updated"],
718
+ );
719
const item = events.find((event) => event.type === "turn_item.updated");
720
const request = events.find((event) => event.type === "runtime_request.updated");
721
assert.equal(item?.type, "turn_item.updated");
packages/provider-opencode/src/server/adapter.ts
+6
-5
@@ -1899,15 +1899,16 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function
1899
},
1900
});
1901
yield* emitProviderEvent({
1902
- type: "runtime_request.updated",
1902
+ type: "turn_item.updated",
1903
driver: OPENCODE_PROVIDER,
1904
- threadId: turn.threadId,
1905
- runtimeRequest,
1904
+ turnItem: runtimeRequestTurnItem(pending, "waiting", null, now),
1905
});
1906
+ // A response can commit as soon as the actionable request is published.
1907
yield* emitProviderEvent({
1908
- type: "turn_item.updated",
1908
+ type: "runtime_request.updated",
1909
driver: OPENCODE_PROVIDER,
1910
- turnItem: runtimeRequestTurnItem(pending, "waiting", null, now),
1910
+ threadId: turn.threadId,
1911
+ runtimeRequest,
1912
});
1913
yield* updateProviderSession("waiting", null);
1914
});
packages/provider-opencode/src/server/v2/adapter.ts
+2
-1
@@ -1888,13 +1888,14 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr
1888
native,
1889
});
1890
yield* emit({ type: "node.updated", driver, node });
1891
+ yield* emit({ type: "turn_item.updated", driver, turnItem: item });
1892
+ // A response can commit as soon as the actionable request is published.
1893
yield* emit({
1894
type: "runtime_request.updated",
1895
driver,
1896
threadId: turn.input.threadId,
1897
runtimeRequest: request,
1898
});
1897
- yield* emit({ type: "turn_item.updated", driver, turnItem: item });
1899
yield* setSessionStatus("waiting", null);
1900
});
1901