@devries-cameron20 / t3code-1 / commits / db51b6719f
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

perf(server): rank queued provider snapshots by focus and combine superseded ones (#39502)

Co-authored-by: Cursor <cursoragent@cursor.com>

Yash Singh committed Nov 19, 2026 at 21:49 UTC db51b6719f116de156401ed264b5d732f35b2c93
11 files changed +472 -100
apps/server/src/orchestration-v2/EventSink.ts
+6
@@ -15,6 +15,7 @@ import {
15 ThreadId,
16 } from "@t3tools/contracts";
17 import * as ProviderCommitBatch from "./ProviderCommitBatch.ts";
18 +import * as ThreadViewers from "./ThreadViewers.ts";
19 import * as Context from "effect/Context";
20 import * as DateTime from "effect/DateTime";
21 import * as Effect from "effect/Effect";
@@ -96,6 +97,7 @@ export interface EventSinkV2Shape {
97 process: (event: A) => Effect.Effect<void, E2, R2>,
98 after: Effect.Effect<boolean, E2, R2>,
99 canBatch: Effect.Effect<boolean, never, R2>,
100 + priority?: ProviderCommitBatch.StreamPriority,
101 ) => Effect.Effect<
102 void,
103 E | E2 | ProviderCommitBatch.CommitBatchError | Schema.SchemaError,
@@ -277,10 +279,12 @@ const layerBase: Layer.Layer<
279 | ProjectStore.ProjectStoreV2
280 | SqlClient.SqlClient
281 | TurnItemPositionStore.TurnItemPositionStoreV2
282 + | ThreadViewers.ThreadViewers
283 > = Layer.effect(
284 EventSinkV2,
285 Effect.gen(function* () {
286 const sql = yield* SqlClient.SqlClient;
287 + const threadViewers = yield* ThreadViewers.ThreadViewers;
288 const commandReceipts = yield* CommandReceiptStore.CommandReceiptStoreV2;
289 const effectOutbox = yield* EffectOutbox.EffectOutboxV2;
290 const eventStore = yield* EventStore.EventStoreV2;
@@ -1174,6 +1178,7 @@ const layerBase: Layer.Layer<
1178 ),
1179 );
1180 }),
1181 + threadViewers.isViewed,
1182 );
1183
1184 return EventSinkV2.of({
@@ -1327,6 +1332,7 @@ export const layer: Layer.Layer<
1332 EffectOutbox.layer,
1333 ProjectStore.layer,
1334 TurnItemPositionStore.layer,
1335 + ThreadViewers.layer,
1336 ),
1337 ),
1338 );
apps/server/src/orchestration-v2/FoundationPersistence.test.ts
+10 -1
@@ -64,6 +64,7 @@ import * as CommandReceiptStore from "./CommandReceiptStore.ts";
64 import * as EffectOutbox from "./EffectOutbox.ts";
65 import * as EffectWorker from "./EffectWorker.ts";
66 import * as EventSink from "./EventSink.ts";
67 +import * as ThreadViewers from "./ThreadViewers.ts";
68 import * as EventStore from "./EventStore.ts";
69 import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
70 import * as ProjectionMaintenance from "./ProjectionMaintenance.ts";
@@ -491,7 +492,13 @@ it.effect("commits other threads' writes ahead of a thread's queued provider sna
492 .pipe(Effect.forkScoped({ startImmediately: true }));
493 yield* Queue.offer(input, snapshot);
494 yield* Deferred.await(started);
494 - yield* Queue.offerAll(input, [snapshot, snapshot]);
495 + yield* Queue.offerAll(
496 + input,
497 + [2, 3].map((n) => ({
498 + ...snapshot,
499 + node: { ...busyNode.payload, id: NodeId.make(`node:busy:${n}`) },
500 + })),
501 + );
502 yield* Deferred.await(pulled);
503 yield* TestClock.adjust(0);
504 const writeTo = (event: OrchestrationV2DomainEvent, id: string) =>
@@ -4310,6 +4317,7 @@ it.effect.each(["direct", "provider-batch", "cancel-provider-batch"] as const)(
4317 layerCommandReceiptStoreProvided,
4318 ProjectStore.layer.pipe(Layer.provide(layerDatabase)),
4319 TurnItemPositionStore.layer.pipe(Layer.provide(layerDatabase)),
4320 + ThreadViewers.layer,
4321 ),
4322 ),
4323 );
@@ -4608,6 +4616,7 @@ it.effect.each(["cancel transaction", "fail transaction", "cancel publication"]
4616 layerCommandReceiptStoreProvided,
4617 ProjectStore.layer.pipe(Layer.provide(layerDatabase)),
4618 TurnItemPositionStore.layer.pipe(Layer.provide(layerDatabase)),
4619 + ThreadViewers.layer,
4620 ),
4621 ),
4622 );
apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts
+157 -1
@@ -21,7 +21,7 @@ function event(index: number, text = `delta:${index}`): ProviderAdapter.Provider
21 type: "message.updated",
22 driver,
23 message: {
24 - id: MessageId.make("single-message"),
24 + id: MessageId.make(`message:${index}`),
25 threadId,
26 runId: null,
27 nodeId: null,
@@ -36,6 +36,15 @@ function event(index: number, text = `delta:${index}`): ProviderAdapter.Provider
36 },
37 };
38 }
39 +function snapshot(id: string, text: string, threadId = "single-thread") {
40 + const base = event(0, text);
41 + return base.type === "message.updated"
42 + ? {
43 + ...base,
44 + message: { ...base.message, id: MessageId.make(id), threadId: ThreadId.make(threadId) },
45 + }
46 + : base;
47 +}
48 const control: ProviderAdapter.ProviderAdapterV2Event = {
49 type: "background_work.updated",
50 driver,
@@ -45,6 +54,7 @@ const control: ProviderAdapter.ProviderAdapterV2Event = {
54
55 const recordBatches = Effect.fn("test.recordProviderBatches")(function* (
56 beforeCommit: Effect.Effect<void> = Effect.void,
57 + isViewed?: (threadId: string) => boolean,
58 ) {
59 const output = yield* Queue.unbounded<ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event>>();
60 let current: ProviderAdapter.ProviderAdapterV2Event[] = [];
@@ -57,6 +67,8 @@ const recordBatches = Effect.fn("test.recordProviderBatches")(function* (
67 if (current.length > 0) yield* Queue.offer(output, current);
68 }),
69 true,
70 + undefined,
71 + isViewed,
72 );
73 const process = (raw: ProviderAdapter.ProviderAdapterV2Event) =>
74 raw.type === "background_work.updated"
@@ -463,6 +475,150 @@ it.effect.each(["none", "writer", "snapshots"] as const)(
475 }),
476 );
477
478 +const blockFirstCommit = Effect.gen(function* () {
479 + const started = yield* Deferred.make<void>();
480 + const release = yield* Deferred.make<void>();
481 + let first = true;
482 + const block = Effect.gen(function* () {
483 + if (!first) return;
484 + first = false;
485 + yield* Deferred.succeed(started, undefined);
486 + yield* Deferred.await(release);
487 + });
488 + return { started, release, block };
489 +});
490 +
491 +it.effect.each([
492 + { keys: ["a", "b", "a", "b", "a", "b"], kept: [0, 4, 5] },
493 + // A first appearance between two snapshots of an entity keeps the earlier snapshot.
494 + { keys: ["a", "b", "a", "b", "c", "a", "b"], kept: [0, 2, 3, 4, 5, 6] },
495 +])("combines superseded snapshots queued behind a busy commit: $keys", ({ keys, kept }) =>
496 + Effect.gen(function* () {
497 + const writer = yield* blockFirstCommit;
498 + const recorder = yield* recordBatches(writer.block);
499 + const leading = yield* recorder
500 + .consume(Stream.make(event(-1)), recorder.process, Effect.succeed(false))
501 + .pipe(Effect.forkScoped({ startImmediately: true }));
502 + yield* Deferred.await(writer.started);
503 + const snapshots = keys.map((key, index) => snapshot(key, `${key}:${index}`));
504 + const pulled = yield* Deferred.make<void>();
505 + const consumer = yield* recorder
506 + .consume(
507 + Stream.fromIterable(snapshots).pipe(
508 + Stream.tap((raw) =>
509 + raw === snapshots.at(-1) ? Deferred.succeed(pulled, undefined) : Effect.void,
510 + ),
511 + ),
512 + recorder.process,
513 + Effect.succeed(false),
514 + )
515 + .pipe(Effect.forkScoped({ startImmediately: true }));
516 + yield* Deferred.await(pulled);
517 + yield* TestClock.adjust(0);
518 + yield* Deferred.succeed(writer.release, undefined);
519 + yield* Fiber.join(leading);
520 + yield* Fiber.join(consumer);
521 + assert.deepEqual(yield* Queue.takeAll(recorder.output), [
522 + [event(-1)],
523 + kept.map((index) => snapshots[index]!),
524 + ]);
525 + }),
526 +);
527 +
528 +it.effect("makes streams wait to read more once the shared backlog is full", () =>
529 + Effect.gen(function* () {
530 + const writer = yield* blockFirstCommit;
531 + const recorder = yield* recordBatches(writer.block);
532 + const leading = yield* recorder
533 + .consume(Stream.make(event(-1)), recorder.process, Effect.succeed(false))
534 + .pipe(Effect.forkScoped({ startImmediately: true }));
535 + yield* Deferred.await(writer.started);
536 + let arrivals = 0;
537 + const streams = Array.from({ length: 5 }, (_, stream) =>
538 + Array.from({ length: 64 }, (_, n) => snapshot(`${stream}:${n}`, `${stream}:${n}`)),
539 + );
540 + const consumers = yield* Effect.forEach(streams, (snapshots) =>
541 + recorder
542 + .consume(
543 + Stream.fromIterable(snapshots).pipe(
544 + Stream.tap(() =>
545 + Effect.sync(() => {
546 + arrivals++;
547 + }),
548 + ),
549 + ),
550 + recorder.process,
551 + Effect.succeed(false),
552 + )
553 + .pipe(Effect.forkScoped({ startImmediately: true })),
554 + );
555 + yield* TestClock.adjust(0);
556 + // 256 queued, plus at most one read per stream waiting for room.
557 + assert.isAtMost(arrivals, 256 + streams.length);
558 + yield* Deferred.succeed(writer.release, undefined);
559 + yield* Fiber.join(leading);
560 + yield* Effect.forEach(consumers, Fiber.join, { discard: true });
561 + assert.lengthOf((yield* Queue.takeAll(recorder.output)).flat(), 1 + 5 * 64);
562 + }),
563 +);
564 +
565 +it.effect.each([false, true])(
566 + "orders a backlog by focus, thread kind and a finishing control with starved=%s",
567 + (starved) =>
568 + Effect.gen(function* () {
569 + const writer = yield* blockFirstCommit;
570 + const recorder = yield* recordBatches(writer.block, (threadId) => threadId === "focused");
571 + const leading = yield* recorder
572 + .consume(Stream.make(event(-1)), recorder.process, Effect.succeed(false))
573 + .pipe(Effect.forkScoped({ startImmediately: true }));
574 + yield* Deferred.await(writer.started);
575 + const streams = [
576 + { name: "subagent", delegated: true, finishing: false },
577 + { name: "main", delegated: false, finishing: false },
578 + { name: "finishing", delegated: false, finishing: true },
579 + { name: "focused", delegated: false, finishing: false },
580 + ];
581 + const pulled = yield* Deferred.make<void>();
582 + let arrivals = 0;
583 + const consumers = yield* Effect.forEach(streams, ({ name, delegated, finishing }) =>
584 + recorder
585 + .consume(
586 + // Open streams, so only the control marks a stream as finishing; EOF would too.
587 + Stream.concat(
588 + Stream.make(snapshot(name, name, name), ...(finishing ? [control] : [])),
589 + Stream.never,
590 + ).pipe(
591 + Stream.tap(() =>
592 + ++arrivals === streams.length + 1
593 + ? Deferred.succeed(pulled, undefined)
594 + : Effect.void,
595 + ),
596 + ),
597 + recorder.process,
598 + Effect.succeed(false),
599 + Effect.succeed(true),
600 + { threadId: name, delegated },
601 + )
602 + .pipe(Effect.forkScoped({ startImmediately: true })),
603 + );
604 + yield* Deferred.await(pulled);
605 + yield* TestClock.adjust(starved ? "30 seconds" : 0);
606 + yield* Deferred.succeed(writer.release, undefined);
607 + yield* Fiber.join(leading);
608 + const outputs = yield* Queue.takeN(recorder.output, 3);
609 + yield* Effect.forEach(consumers, Fiber.interrupt, { discard: true });
610 + // Waiting 30 seconds ranks a subagent like a main thread.
611 + const order = starved
612 + ? ["focused", "finishing", "subagent", "main"]
613 + : ["focused", "finishing", "main", "subagent"];
614 + assert.deepEqual(Array.from(outputs), [
615 + [event(-1)],
616 + order.map((name) => snapshot(name, name, name)),
617 + [control],
618 + ]);
619 + }),
620 +);
621 +
622 it.effect.each(["failure", "stop"] as const)(
623 "flushes the prefix and respects a source %s barrier",
624 (mode) =>
apps/server/src/orchestration-v2/ProviderCommitBatch.ts
+249 -96
@@ -7,12 +7,17 @@ import * as Deferred from "effect/Deferred";
7 import * as Exit from "effect/Exit";
8 import * as Queue from "effect/Queue";
9 import * as Schema from "effect/Schema";
10 +import * as Semaphore from "effect/Semaphore";
11 import * as Stream from "effect/Stream";
12
13 const MAX_EVENTS = 64;
14 const MAX_BYTES = 1024 * 1024;
15 // While another write waits, a shared transaction stops taking jobs after this long.
16 const MAX_HOLD_NANOS = 100_000_000n;
17 +// A stream waiting this long ranks like a main thread, so subagents cannot starve.
18 +const STARVATION_MILLIS = 30_000;
19 +// Bounds snapshots held across stream queues; a full backlog makes streams wait to read more.
20 +const MAX_QUEUED = 4 * MAX_EVENTS;
21 const encodeQueue = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown));
22
23 export const enabled = Config.Boolean("T3CODE_PROVIDER_COMMIT_BATCHING").pipe(
@@ -21,35 +26,66 @@ export const enabled = Config.Boolean("T3CODE_PROVIDER_COMMIT_BATCHING").pipe(
26 );
27
28 type Snapshot =
24 - | { readonly type: "node.updated"; readonly node: { readonly threadId: string } }
25 - | { readonly type: "message.updated"; readonly message: { readonly threadId: string } }
26 - | { readonly type: "turn_item.updated"; readonly turnItem: { readonly threadId: string } }
27 - | { readonly type: "plan.updated"; readonly plan: { readonly threadId: string } };
29 + | {
30 + readonly type: "node.updated";
31 + readonly node: { readonly id: string; readonly threadId: string };
32 + }
33 + | {
34 + readonly type: "message.updated";
35 + readonly message: { readonly id: string; readonly threadId: string };
36 + }
37 + | {
38 + readonly type: "turn_item.updated";
39 + readonly turnItem: { readonly id: string; readonly threadId: string };
40 + }
41 + | {
42 + readonly type: "plan.updated";
43 + readonly plan: { readonly id: string; readonly threadId: string };
44 + };
45 const isSnapshot = (event: { readonly type: string }): event is Snapshot =>
46 event.type === "node.updated" ||
47 event.type === "message.updated" ||
48 event.type === "turn_item.updated" ||
49 event.type === "plan.updated";
33 -const snapshotThreadId = (event: Snapshot) => {
34 - switch (event.type) {
35 - case "node.updated":
36 - return event.node.threadId;
37 - case "message.updated":
38 - return event.message.threadId;
39 - case "turn_item.updated":
40 - return event.turnItem.threadId;
41 - case "plan.updated":
42 - return event.plan.threadId;
43 - }
50 +const snapshotEntity = (event: Snapshot) => {
51 + const entity =
52 + event.type === "node.updated"
53 + ? event.node
54 + : event.type === "message.updated"
55 + ? event.message
56 + : event.type === "turn_item.updated"
57 + ? event.turnItem
58 + : event.plan;
59 + return { key: `${event.type}:${entity.id}`, threadId: entity.threadId };
60 };
61
62 +/** Who a provider stream writes for; ranks its snapshots against other streams'. */
63 +export interface StreamPriority {
64 + readonly threadId: string;
65 + /** A subagent or delegated task, which yields to main threads. */
66 + readonly delegated: boolean;
67 +}
68 +
69 export class CommitBatchError extends Schema.TaggedError<CommitBatchError>()("CommitBatchError", {
70 cause: Schema.Defect(),
71 }) {}
72 export const isCommitBatchError = Schema.is(CommitBatchError);
73
51 -interface Job {
74 +interface StreamState {
75 readonly owner: { active: boolean };
76 + readonly priority: StreamPriority | undefined;
77 + /** Queued snapshots not yet taken into a batch, in stream order. */
78 + readonly jobs: Array<Job>;
79 + unsettled: number;
80 + servedAt: number;
81 + /** A control, failure or EOF of this stream waits for its queued snapshots. */
82 + finishing: boolean;
83 +}
84 +interface Job {
85 + readonly stream: StreamState;
86 + /** Still in its stream's queue, holding one of the backlog's `MAX_QUEUED` slots. */
87 + queued: boolean;
88 + readonly key: string;
89 readonly threadId: string;
90 readonly queuedAt: number;
91 readonly bytes: number;
@@ -58,34 +94,51 @@ interface Job {
94 exit: Exit.Exit<void, CommitBatchError>,
95 preserveFailure: boolean,
96 ) => Effect.Effect<void>;
97 + /** Settles a job a later snapshot of the same entity replaced, with that snapshot's outcome. */
98 + readonly skip: (exit: Exit.Exit<void, CommitBatchError>) => Effect.Effect<void>;
99 readonly cancel: Effect.Effect<void>;
100 readonly settled: Effect.Effect<void>;
101 }
102 +interface Unit {
103 + readonly job: Job;
104 + readonly superseded: ReadonlyArray<Job>;
105 +}
106 interface Batch {
65 - readonly events: ReadonlyArray<Job>;
107 + readonly units: ReadonlyArray<Unit>;
108 readonly reason: "ready" | "limit";
109 + readonly dequeued: number;
110 }
111
69 -function collect(queue: Queue.Queue<Job>) {
70 - return Stream.fromEffectRepeat(Queue.takeBetween(queue, 1, MAX_EVENTS)).pipe(
71 - Stream.flatMap((ready) => {
72 - const batches: Array<Batch> = [];
73 - let events: Array<Job> = [];
74 - let bytes = 0;
75 - const flush = (reason: Batch["reason"]) => {
76 - if (events.length > 0) batches.push({ events, reason });
77 - events = [];
78 - bytes = 0;
79 - };
80 - for (const job of ready) {
81 - if (events.length > 0 && bytes + job.bytes > MAX_BYTES) flush("limit");
82 - events.push(job);
83 - bytes += job.bytes;
84 - if (events.length >= MAX_EVENTS || bytes >= MAX_BYTES) flush("limit");
85 - }
86 - flush("ready");
87 - return Stream.fromArray(batches);
88 - }),
112 +/**
113 + * Snapshots carry an entity's full state, so an earlier one is dropped when a later one of the
114 + * same entity follows it in this stream with no first appearance of another entity in between.
115 + * Kept jobs stay in stream order, and entities first appear in the same order as before.
116 + */
117 +function coalesce(jobs: ReadonlyArray<Job>): Array<Unit> {
118 + const firstSeen = new Map<string, number>();
119 + jobs.forEach((job, index) => {
120 + if (!firstSeen.has(job.key)) firstSeen.set(job.key, index);
121 + });
122 + const survivor: Array<number> = Array.from({ length: jobs.length }, (_, index) => index);
123 + const nextOfKey = new Map<string, number>();
124 + for (let index = jobs.length - 1; index >= 0; index--) {
125 + const job = jobs[index]!;
126 + const next = nextOfKey.get(job.key);
127 + nextOfKey.set(job.key, index);
128 + if (next === undefined) continue;
129 + let replaced = true;
130 + for (let between = index + 1; replaced && between < next; between++) {
131 + if (firstSeen.get(jobs[between]!.key)! > index) replaced = false;
132 + }
133 + if (replaced) survivor[index] = survivor[next]!;
134 + }
135 + const superseded = new Map<number, Array<Job>>();
136 + jobs.forEach((job, index) => {
137 + const kept = survivor[index]!;
138 + if (kept !== index) superseded.set(kept, [...(superseded.get(kept) ?? []), job]);
139 + });
140 + return jobs.flatMap((job, index) =>
141 + survivor[index] === index ? [{ job, superseded: superseded.get(index) ?? [] }] : [],
142 );
143 }
144
@@ -102,32 +155,44 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* (
155 isolate: (
156 work: Effect.Effect<void, CommitBatchError>,
157 ) => Effect.Effect<void, CommitBatchError> = (work) => work,
158 + /** Whether a client has the thread open; its streams rank first. */
159 + isViewed: (threadId: string) => boolean = () => false,
160 ) {
106 - const queue = yield* Queue.bounded<Job>(MAX_EVENTS);
161 + const wake = yield* Queue.sliding<void>(1);
162 + const backlog = yield* Semaphore.make(MAX_QUEUED);
163 + const streams = new Set<StreamState>();
164 const pending = new Set<Job>();
165 let closed = false;
166 + let servings = 0;
167 // Writes in `barrier` that have not finished; they wait on the batch or the writer it holds.
168 let waitingWriters = 0;
169 const shutdown = Effect.gen(function* () {
170 closed = true;
113 - yield* Queue.shutdown(queue);
171 + yield* Queue.shutdown(wake);
172 for (const job of pending) yield* job.cancel;
173 + yield* backlog.releaseAll;
174 });
175 yield* Effect.addFinalizer(() => shutdown);
176 // These helpers run once per raw event; the shared transaction owns the tracing span.
177 const enqueue = Effect.fnUntraced(function* <A, E, R>(
178 effect: Effect.Effect<A, E, R>,
120 - token: { active: boolean },
121 - threadId: string,
179 + stream: StreamState,
180 + entity: { readonly key: string; readonly threadId: string },
181 bytes: number,
182 + skipped: A,
183 ) {
184 const caller = yield* Effect.context<R>();
185 const reply = yield* Deferred.make<A, E | CommitBatchError>();
186 let ownFailure: Cause.Cause<E> | undefined;
187 let result: A;
188 + const release = () => {
189 + if (pending.delete(job)) stream.unsettled--;
190 + };
191 const job: Job = {
129 - owner: token,
130 - threadId,
192 + stream,
193 + queued: false,
194 + key: entity.key,
195 + threadId: entity.threadId,
196 queuedAt: yield* Clock.currentTimeMillis,
197 bytes,
198 execute: effect.pipe(
@@ -149,44 +214,111 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* (
214 exit: Exit.Exit<void, CommitBatchError>,
215 preserveFailure: boolean,
216 ) {
152 - pending.delete(job);
217 + release();
218 if (Exit.isFailure(exit)) {
154 - token.active = false;
219 + stream.owner.active = false;
220 yield* Deferred.failCause(
221 reply,
222 preserveFailure && ownFailure !== undefined ? ownFailure : exit.cause,
223 );
224 } else yield* Deferred.succeed(reply, result);
225 }),
161 - cancel: Effect.gen(function* () {
162 - pending.delete(job);
163 - yield* Deferred.interrupt(reply);
226 + skip: (exit) =>
227 + Exit.isFailure(exit)
228 + ? job.settle(exit, false)
229 + : Effect.suspend(() => {
230 + release();
231 + return Deferred.succeed(reply, skipped);
232 + }),
233 + cancel: Effect.suspend(() => {
234 + release();
235 + const dequeued = job.queued;
236 + job.queued = false;
237 + return Deferred.interrupt(reply).pipe(
238 + Effect.andThen(dequeued ? backlog.release(1) : Effect.void),
239 + );
240 }),
241 settled: Deferred.await(reply).pipe(Effect.exit, Effect.asVoid),
242 };
243 yield* Effect.uninterruptibleMask((restore) =>
244 Effect.gen(function* () {
245 + if (closed) return yield* Deferred.interrupt(reply);
246 + yield* restore(backlog.take(1));
247 + if (closed) {
248 + yield* backlog.release(1);
249 + return yield* Deferred.interrupt(reply);
250 + }
251 + job.queued = true;
252 pending.add(job);
170 - const accepted = yield* restore(Queue.offer(queue, job)).pipe(
171 - Effect.onError(() => job.cancel),
172 - );
173 - if (!accepted) yield* job.cancel;
253 + stream.unsettled++;
254 + stream.jobs.push(job);
255 + yield* Queue.offer(wake, undefined);
256 }),
257 );
258 return Deferred.await(reply);
259 });
178 - // Commits a prefix of `jobs` in one transaction and returns how many it took.
179 - const commitPrefix = (jobs: ReadonlyArray<Job>, reason: Batch["reason"]) =>
260 + const rank = (stream: StreamState, now: number) => {
261 + const tier =
262 + stream.priority === undefined
263 + ? 1
264 + : isViewed(stream.priority.threadId)
265 + ? 0
266 + : stream.priority.delegated
267 + ? 2
268 + : 1;
269 + const starved = now - stream.jobs[0]!.queuedAt >= STARVATION_MILLIS;
270 + return [starved ? Math.min(tier, 1) : tier, stream.finishing ? 0 : 1, stream.servedAt];
271 + };
272 + // Viewed streams first, then main threads, then subagents; within a tier, streams finishing a
273 + // control first, then the stream served longest ago.
274 + const nextBatch = (now: number): Batch | undefined => {
275 + const ready = Array.from(streams, (stream) =>
276 + stream.jobs.length === 0 ? [] : [{ stream, rank: rank(stream, now) }],
277 + )
278 + .flat()
279 + .sort((left, right) => {
280 + for (let index = 0; index < left.rank.length; index++) {
281 + const order = left.rank[index]! - right.rank[index]!;
282 + if (order !== 0) return order;
283 + }
284 + return 0;
285 + });
286 + if (ready.length === 0) return undefined;
287 + const units: Array<Unit> = [];
288 + let bytes = 0;
289 + let dequeued = 0;
290 + let reason: Batch["reason"] = "ready";
291 + for (const { stream } of ready) {
292 + let take = 0;
293 + while (take < stream.jobs.length && take < MAX_EVENTS - units.length) {
294 + const next = stream.jobs[take]!.bytes;
295 + if (units.length + take > 0 && bytes + next > MAX_BYTES) break;
296 + bytes += next;
297 + take++;
298 + }
299 + if (take < stream.jobs.length) reason = "limit";
300 + if (take === 0) break;
301 + const taken = stream.jobs.splice(0, take);
302 + for (const job of taken) job.queued = false;
303 + dequeued += take;
304 + units.push(...coalesce(taken));
305 + stream.servedAt = ++servings;
306 + if (units.length >= MAX_EVENTS || bytes >= MAX_BYTES) break;
307 + }
308 + return { units, reason, dequeued };
309 + };
310 + // Commits a prefix of `units` in one transaction and returns how many it took.
311 + const commitPrefix = (units: ReadonlyArray<Unit>, reason: Batch["reason"]) =>
312 Effect.uninterruptibleMask((restore) =>
313 Effect.gen(function* () {
314 const outcomes = new Map<Job, Exit.Exit<void, CommitBatchError>>();
315 let taken = 0;
316 const work = Effect.gen(function* () {
185 - // A retried transaction runs this again from the first job.
317 + // A retried transaction runs this again from the first unit.
318 outcomes.clear();
319 taken = 0;
320 const startedAt = yield* Clock.monotonicTimeNanos;
189 - for (const job of jobs) {
321 + for (const { job, superseded } of units) {
322 if (
323 taken > 0 &&
324 waitingWriters > 0 &&
@@ -194,63 +326,62 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* (
326 )
327 return;
328 taken++;
197 - if (!job.owner.active) {
329 + if (!job.stream.owner.active) {
330 yield* job.cancel;
331 + for (const replaced of superseded) yield* replaced.cancel;
332 continue;
333 }
201 - const outcome = yield* (jobs.length > 1 ? isolate(job.execute) : job.execute).pipe(
334 + const outcome = yield* (units.length > 1 ? isolate(job.execute) : job.execute).pipe(
335 Effect.exit,
336 );
337 outcomes.set(job, outcome);
338 if (Exit.isFailure(outcome)) {
206 - job.owner.active = false;
207 - if (jobs.length === 1) return yield* Effect.failCause(outcome.cause);
339 + job.stream.owner.active = false;
340 + if (units.length === 1) return yield* Effect.failCause(outcome.cause);
341 }
342 }
343 });
344 const exit = yield* restore(
345 commit(work).pipe(Effect.provideService(CurrentWorker, yield* Effect.fiberId)),
346 ).pipe(Effect.exit);
214 - const committed = taken === 0 ? jobs : jobs.slice(0, taken);
347 + const committed = taken === 0 ? units : units.slice(0, taken);
348 + const jobs = committed.flatMap((unit) => [unit.job, ...unit.superseded]);
349 yield* Effect.annotateCurrentSpan({
350 "orchestration_v2.work_item_count": committed.length,
217 - "orchestration_v2.deferred_work_item_count": jobs.length - committed.length,
218 - "orchestration_v2.consumer_count": new Set(committed.map((job) => job.owner)).size,
219 - "orchestration_v2.raw_json_bytes": committed.reduce((bytes, job) => bytes + job.bytes, 0),
351 + "orchestration_v2.coalesced_work_item_count": jobs.length - committed.length,
352 + "orchestration_v2.deferred_work_item_count": units.length - committed.length,
353 + "orchestration_v2.consumer_count": new Set(jobs.map((job) => job.stream)).size,
354 + "orchestration_v2.raw_json_bytes": jobs.reduce((bytes, job) => bytes + job.bytes, 0),
355 "orchestration_v2.batch_flush_reason": reason,
356 "orchestration_v2.backlog_batching": true,
357 "orchestration_v2.batch_wait_ms":
223 - (yield* Clock.currentTimeMillis) - Math.min(...committed.map((job) => job.queuedAt)),
358 + (yield* Clock.currentTimeMillis) - Math.min(...jobs.map((job) => job.queuedAt)),
359 });
225 - for (const job of committed) {
360 + for (const { job, superseded } of committed) {
361 const outcome = outcomes.get(job) ?? Exit.void;
362 + const final = Exit.isSuccess(exit) ? outcome : exit;
363 yield* job.settle(
228 - Exit.isSuccess(exit) ? outcome : exit,
364 + final,
365 Exit.isSuccess(exit) ||
366 (Exit.isFailure(exit) &&
367 Exit.isFailure(outcome) &&
368 Cause.squash(exit.cause) === Cause.squash(outcome.cause)),
369 );
370 + for (const replaced of superseded) yield* replaced.skip(final);
371 }
372 return committed.length;
373 }),
374 ).pipe(Effect.withSpan("orchestrationV2.ProviderCommitBatch.flush"));
375 if (enabled) {
239 - yield* collect(queue).pipe(
240 - Stream.runForEach(
241 - Effect.fnUntraced(function* (batch: Batch) {
242 - let jobs = batch.events;
243 - while (jobs.length > 0) {
244 - for (const job of jobs) if (!job.owner.active) yield* job.cancel;
245 - jobs = jobs.filter((job) => job.owner.active);
246 - if (jobs.length === 0) return;
247 - jobs = jobs.slice(yield* commitPrefix(jobs, batch.reason));
248 - }
249 - }),
250 - ),
251 - Effect.ensuring(shutdown),
252 - Effect.forkScoped,
253 - );
376 + yield* Effect.forever(
377 + Effect.gen(function* () {
378 + const batch = nextBatch(yield* Clock.currentTimeMillis);
379 + if (batch === undefined) return yield* Queue.take(wake);
380 + yield* backlog.release(batch.dequeued);
381 + let units = batch.units;
382 + while (units.length > 0) units = units.slice(yield* commitPrefix(units, batch.reason));
383 + }),
384 + ).pipe(Effect.ensuring(shutdown), Effect.forkScoped);
385 }
386 /**
387 * Runs `effect` after queued snapshots for `threadIds` commit; omitted, after every queued
@@ -286,6 +417,7 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* (
417 process: (event: A) => Effect.Effect<void, E2, R2>,
418 after: Effect.Effect<boolean, E2, R2>,
419 canBatch: Effect.Effect<boolean, never, R2> = Effect.succeed(true),
420 + priority?: StreamPriority,
421 ) {
422 if (!enabled)
423 return yield* source.pipe(
@@ -293,7 +425,15 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* (
425 Stream.takeUntil(Boolean),
426 Stream.runDrain,
427 );
296 - const token = { active: true };
428 + const stream: StreamState = {
429 + owner: { active: true },
430 + priority,
431 + jobs: [],
432 + unsettled: 0,
433 + servedAt: 0,
434 + finishing: false,
435 + };
436 + streams.add(stream);
437 const signals = source.pipe(
438 Stream.map((event) => ({ _tag: "Event" as const, event })),
439 Stream.catchCause((cause) => Stream.succeed({ _tag: "Failure" as const, cause })),
@@ -309,20 +449,31 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* (
449 const processed = yield* Deferred.make<void>();
450 let reply: Effect.Effect<boolean, E | E2 | CommitBatchError, R2>;
451 let check = false;
312 - if (signal._tag === "End") reply = Effect.succeed(true);
313 - else if (signal._tag === "Failure") reply = Effect.failCause(signal.cause);
314 - else if (isSnapshot(signal.event) && (yield* canBatch)) {
315 - const bytes = Buffer.byteLength(yield* encodeQueue(signal.event));
452 + const snapshot =
453 + signal._tag === "Event" && isSnapshot(signal.event) && (yield* canBatch)
454 + ? signal.event
455 + : undefined;
456 + if (snapshot !== undefined) {
457 + const bytes = Buffer.byteLength(yield* encodeQueue(snapshot));
458 reply = yield* enqueue(
317 - process(signal.event).pipe(Effect.as(false)),
318 - token,
319 - snapshotThreadId(signal.event),
459 + process(snapshot).pipe(Effect.as(false)),
460 + stream,
461 + snapshotEntity(snapshot),
462 bytes,
463 + false,
464 );
465 } else {
323 - reply = process(signal.event).pipe(Effect.as(false));
324 - check = true;
325 - resume = Deferred.await(processed);
466 + if (stream.unsettled > 0) stream.finishing = true;
467 + if (signal._tag === "End") reply = Effect.succeed(true);
468 + else if (signal._tag === "Failure") reply = Effect.failCause(signal.cause);
469 + else {
470 + const event = signal.event;
471 + reply = Effect.sync(() => {
472 + stream.finishing = false;
473 + }).pipe(Effect.andThen(process(event)), Effect.as(false));
474 + check = true;
475 + resume = Deferred.await(processed);
476 + }
477 }
478 return { reply, processed, check };
479 }),
@@ -335,7 +486,7 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* (
486 ),
487 Effect.tap((stop) =>
488 Effect.sync(() => {
338 - if (stop) token.active = false;
489 + if (stop) stream.owner.active = false;
490 }),
491 ),
492 Effect.ensuring(Deferred.succeed(processed, undefined)),
@@ -344,8 +495,10 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* (
495 Stream.takeUntil(Boolean),
496 Stream.runDrain,
497 Effect.ensuring(
347 - Effect.sync(() => {
348 - token.active = false;
498 + Effect.gen(function* () {
499 + stream.owner.active = false;
500 + streams.delete(stream);
501 + for (const job of stream.jobs.splice(0)) yield* job.cancel;
502 }),
503 ),
504 );
apps/server/src/orchestration-v2/RunExecutionService.ts
+2
@@ -1,6 +1,7 @@
1 import { makeAssistantStreamingFilter } from "./assistantStreaming.ts";
2 import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
3 import {
4 + isDelegatedAppThread,
5 isOrchestrationV2WorkActive,
6 CommandId,
7 type EventId,
@@ -1988,6 +1989,7 @@ export const layer: Layer.Layer<
1989 return yield* shouldStopProviderEventIngestion;
1990 }),
1991 Ref.get(rootTerminalSeen).pipe(Effect.map((seen) => !seen)),
1992 + { threadId: input.run.threadId, delegated: isDelegatedAppThread(input.appThread) },
1993 )
1994 .pipe(
1995 Effect.ensuring(closeIdleFlush),
apps/server/src/orchestration-v2/ThreadViewers.ts new
+33
@@ -0,0 +1,33 @@
1 +import * as Context from "effect/Context";
2 +import * as Effect from "effect/Effect";
3 +import * as Layer from "effect/Layer";
4 +import type * as Scope from "effect/Scope";
5 +
6 +export class ThreadViewers extends Context.Service<
7 + ThreadViewers,
8 + {
9 + /** Counts a client viewing `threadId` until the enclosing scope closes. */
10 + readonly view: (threadId: string) => Effect.Effect<void, never, Scope.Scope>;
11 + readonly isViewed: (threadId: string) => boolean;
12 + }
13 +>()("t3/orchestration-v2/ThreadViewers") {}
14 +
15 +export const layer = Layer.sync(ThreadViewers, () => {
16 + const viewers = new Map<string, number>();
17 + return ThreadViewers.of({
18 + view: Effect.fn("ThreadViewers.view")(function* (threadId: string) {
19 + yield* Effect.acquireRelease(
20 + Effect.sync(() => {
21 + viewers.set(threadId, (viewers.get(threadId) ?? 0) + 1);
22 + }),
23 + () =>
24 + Effect.sync(() => {
25 + const remaining = (viewers.get(threadId) ?? 1) - 1;
26 + if (remaining > 0) viewers.set(threadId, remaining);
27 + else viewers.delete(threadId);
28 + }),
29 + );
30 + }),
31 + isViewed: (threadId) => viewers.has(threadId),
32 + });
33 +});
apps/server/src/orchestration-v2/runtimeLayer.ts
+5 -1
@@ -28,6 +28,7 @@ import * as ContextHandoffService from "./ContextHandoffService.ts";
28 import * as EffectOutbox from "./EffectOutbox.ts";
29 import * as EffectWorker from "./EffectWorker.ts";
30 import * as EventSink from "./EventSink.ts";
31 +import * as ThreadViewers from "./ThreadViewers.ts";
32 import * as EventStore from "./EventStore.ts";
33 import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
34 import * as LegacyV1ThreadImporter from "./legacy/LegacyV1ThreadImporter.ts";
@@ -86,7 +87,10 @@ const layerStores = Layer.mergeAll(
87 TurnItemPositionStore.layer,
88 );
89
89 -export const layerEventSink = EventSink.layerFromStores.pipe(Layer.provide(layerStores));
90 +// The server provides this same layer to thread subscriptions; memoized, they share one registry.
91 +export const layerEventSink = EventSink.layerFromStores.pipe(
92 + Layer.provide(Layer.merge(layerStores, ThreadViewers.layer)),
93 +);
94 const layerProjectionMaintenanceProvided = ProjectionMaintenance.layer.pipe(
95 Layer.provide(layerStores),
96 );
apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts
+2 -1
@@ -37,6 +37,7 @@ import * as ContextHandoffService from "../ContextHandoffService.ts";
37 import * as EffectOutbox from "../EffectOutbox.ts";
38 import * as EffectWorker from "../EffectWorker.ts";
39 import * as EventSink from "../EventSink.ts";
40 +import * as ThreadViewers from "../ThreadViewers.ts";
41 import * as EventStore from "../EventStore.ts";
42 import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
43 import * as Orchestrator from "../Orchestrator.ts";
@@ -349,7 +350,7 @@ export function layerWithRegistry<Error>(
350 TurnItemPositionStore.layer,
351 ).pipe(Layer.provide(layerDatabase));
352 const layerEventSinkProvided = EventSink.layerFromStores.pipe(
352 - Layer.provide(Layer.mergeAll(layerStores, layerDatabase)),
353 + Layer.provide(Layer.mergeAll(layerStores, layerDatabase, ThreadViewers.layer)),
354 );
355 const layerRuntime = RuntimePolicy.layerEnforcingCeiling.pipe(
356 Layer.provide(Layer.merge(layerRuntimeBase, layerServerSettings)),
apps/server/src/server.ts
+3
@@ -215,6 +215,7 @@ import * as CursorUsageAccounts from "@t3tools/provider-cursor/server/CursorUsag
215 import * as UsageService from "./usage/UsageService.ts";
216 import * as CodexUsageHistory from "./usage/CodexUsageHistory.ts";
217 import * as RuntimeLayer from "./orchestration-v2/runtimeLayer.ts";
218 +import * as ThreadViewers from "./orchestration-v2/ThreadViewers.ts";
219 import * as ProjectStore from "./orchestration-v2/ProjectStore.ts";
220 import * as ThreadSearch from "./orchestration-v2/ThreadSearch.ts";
221 import * as ThreadSettlementService from "./orchestration-v2/ThreadSettlementService.ts";
@@ -814,6 +815,8 @@ const layerRuntimeCoreDependencies = layerRuntimeCoreDependenciesBase.pipe(
815 const layerRuntimeDependencies = layerRuntimeCoreDependencies.pipe(
816 Layer.provideMerge(layerUsage),
817 Layer.provideMerge(layerBackground),
818 + // The same layer the event sink uses; memoized, subscriptions and batch ranking share it.
819 + Layer.provideMerge(ThreadViewers.layer),
820 Layer.provideMerge(layerResourceDiagnostics),
821 Layer.provideMerge(TraceDiagnostics.layer),
822 Layer.provideMerge(TriageService.layer.pipe(Layer.provide(GitVcsDriver.layer))),
apps/server/src/ws.test.ts
+2
@@ -82,6 +82,7 @@ import * as BackgroundPolicy from "./background/BackgroundPolicy.ts";
82 import * as CheckpointDiffQuery from "./checkpointing/CheckpointDiffQuery.ts";
83 import * as NativeChildWorkQuery from "./orchestration-v2/NativeChildWorkQuery.ts";
84 import * as CommandOutputHub from "./orchestration-v2/CommandOutputHub.ts";
85 +import * as ThreadViewers from "./orchestration-v2/ThreadViewers.ts";
86 import * as CodexInstallation from "./provider/CodexInstallation.ts";
87 import * as DeviceService from "./device/DeviceService.ts";
88 import * as DocumentText from "./provider/DocumentText.ts";
@@ -832,6 +833,7 @@ const layerUnusedConfigRpcServices = Layer.mergeAll(
833 Layer.mock(CheckpointDiffQuery.CheckpointDiffQuery)({}),
834 Layer.mock(NativeChildWorkQuery.NativeChildWorkQuery)({}),
835 Layer.mock(CommandOutputHub.CommandOutputHub)({}),
836 + ThreadViewers.layer,
837 Layer.mock(CodexInstallation.CodexInstallation)({ managedDirectory: "/unused" }),
838 Layer.mock(DeviceService.DeviceService)({}),
839 Layer.mock(EnvironmentTheme.EnvironmentThemeService)({}),
apps/server/src/ws.ts
+3
@@ -119,6 +119,7 @@ import * as EnvironmentTheme from "./environmentTheme.ts";
119 import * as Keybindings from "./keybindings.ts";
120 import * as ExternalLauncher from "./process/externalLauncher.ts";
121 import * as ThreadManagementService from "./orchestration-v2/ThreadManagementService.ts";
122 +import * as ThreadViewers from "./orchestration-v2/ThreadViewers.ts";
123 import * as CommandOutputHub from "./orchestration-v2/CommandOutputHub.ts";
124 import * as McpAppRequests from "./mcpApps/McpAppRequests.ts";
125 import * as ProviderSessionManager from "./orchestration-v2/ProviderSessionManager.ts";
@@ -1480,6 +1481,7 @@ export const layerWsRpc = (
1481 };
1482 const threadLaunch = yield* ThreadLaunchService.ThreadLaunchService;
1483 const commandOutputHub = yield* CommandOutputHub.CommandOutputHub;
1484 + const threadViewers = yield* ThreadViewers.ThreadViewers;
1485 const providerSessionManager = yield* ProviderSessionManager.ProviderSessionManagerV2;
1486 const scheduledTasks = yield* ScheduledTasks.ScheduledTaskService;
1487 const secretRequests = yield* SecretRequests.SecretRequests;
@@ -2322,6 +2324,7 @@ export const layerWsRpc = (
2324 [ORCHESTRATION_V2_WS_METHODS.subscribeThread]: (input) =>
2325 Stream.unwrap(
2326 Effect.annotateCurrentSpan({ "orchestration_v2.thread_id": input.threadId }).pipe(
2327 + Effect.andThen(threadViewers.view(input.threadId)),
2328 Effect.andThen(subscribeOrchestrationV2Thread(input)),
2329 ),
2330 ),