The best way to control Claude Code, Codex and more from any computer or phone
perf(server): rank queued provider snapshots by focus and combine superseded ones (#39502)
Co-authored-by: Cursor <cursoragent@cursor.com>
Yash Singh committed
Nov 19, 2026 at 21:49 UTC
db51b6719f116de156401ed264b5d732f35b2c93
11 files changed
+472
-100
apps/server/src/orchestration-v2/EventSink.ts
+6
@@ -15,6 +15,7 @@ import {
15
ThreadId,
16
} from "@t3tools/contracts";
17
import * as ProviderCommitBatch from "./ProviderCommitBatch.ts";
18
+import * as ThreadViewers from "./ThreadViewers.ts";
19
import * as Context from "effect/Context";
20
import * as DateTime from "effect/DateTime";
21
import * as Effect from "effect/Effect";
@@ -96,6 +97,7 @@ export interface EventSinkV2Shape {
97
process: (event: A) => Effect.Effect<void, E2, R2>,
98
after: Effect.Effect<boolean, E2, R2>,
99
canBatch: Effect.Effect<boolean, never, R2>,
100
+ priority?: ProviderCommitBatch.StreamPriority,
101
) => Effect.Effect<
102
void,
103
E | E2 | ProviderCommitBatch.CommitBatchError | Schema.SchemaError,
@@ -277,10 +279,12 @@ const layerBase: Layer.Layer<
279
| ProjectStore.ProjectStoreV2
280
| SqlClient.SqlClient
281
| TurnItemPositionStore.TurnItemPositionStoreV2
282
+ | ThreadViewers.ThreadViewers
283
> = Layer.effect(
284
EventSinkV2,
285
Effect.gen(function* () {
286
const sql = yield* SqlClient.SqlClient;
287
+ const threadViewers = yield* ThreadViewers.ThreadViewers;
288
const commandReceipts = yield* CommandReceiptStore.CommandReceiptStoreV2;
289
const effectOutbox = yield* EffectOutbox.EffectOutboxV2;
290
const eventStore = yield* EventStore.EventStoreV2;
@@ -1174,6 +1178,7 @@ const layerBase: Layer.Layer<
1178
),
1179
);
1180
}),
1181
+ threadViewers.isViewed,
1182
);
1183
1184
return EventSinkV2.of({
@@ -1327,6 +1332,7 @@ export const layer: Layer.Layer<
1332
EffectOutbox.layer,
1333
ProjectStore.layer,
1334
TurnItemPositionStore.layer,
1335
+ ThreadViewers.layer,
1336
),
1337
),
1338
);
apps/server/src/orchestration-v2/FoundationPersistence.test.ts
+10
-1
@@ -64,6 +64,7 @@ import * as CommandReceiptStore from "./CommandReceiptStore.ts";
64
import * as EffectOutbox from "./EffectOutbox.ts";
65
import * as EffectWorker from "./EffectWorker.ts";
66
import * as EventSink from "./EventSink.ts";
67
+import * as ThreadViewers from "./ThreadViewers.ts";
68
import * as EventStore from "./EventStore.ts";
69
import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
70
import * as ProjectionMaintenance from "./ProjectionMaintenance.ts";
@@ -491,7 +492,13 @@ it.effect("commits other threads' writes ahead of a thread's queued provider sna
492
.pipe(Effect.forkScoped({ startImmediately: true }));
493
yield* Queue.offer(input, snapshot);
494
yield* Deferred.await(started);
494
- yield* Queue.offerAll(input, [snapshot, snapshot]);
495
+ yield* Queue.offerAll(
496
+ input,
497
+ [2, 3].map((n) => ({
498
+ ...snapshot,
499
+ node: { ...busyNode.payload, id: NodeId.make(`node:busy:${n}`) },
500
+ })),
501
+ );
502
yield* Deferred.await(pulled);
503
yield* TestClock.adjust(0);
504
const writeTo = (event: OrchestrationV2DomainEvent, id: string) =>
@@ -4310,6 +4317,7 @@ it.effect.each(["direct", "provider-batch", "cancel-provider-batch"] as const)(
4317
layerCommandReceiptStoreProvided,
4318
ProjectStore.layer.pipe(Layer.provide(layerDatabase)),
4319
TurnItemPositionStore.layer.pipe(Layer.provide(layerDatabase)),
4320
+ ThreadViewers.layer,
4321
),
4322
),
4323
);
@@ -4608,6 +4616,7 @@ it.effect.each(["cancel transaction", "fail transaction", "cancel publication"]
4616
layerCommandReceiptStoreProvided,
4617
ProjectStore.layer.pipe(Layer.provide(layerDatabase)),
4618
TurnItemPositionStore.layer.pipe(Layer.provide(layerDatabase)),
4619
+ ThreadViewers.layer,
4620
),
4621
),
4622
);
apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts
+157
-1
@@ -21,7 +21,7 @@ function event(index: number, text = `delta:${index}`): ProviderAdapter.Provider
21
type: "message.updated",
22
driver,
23
message: {
24
- id: MessageId.make("single-message"),
24
+ id: MessageId.make(`message:${index}`),
25
threadId,
26
runId: null,
27
nodeId: null,
@@ -36,6 +36,15 @@ function event(index: number, text = `delta:${index}`): ProviderAdapter.Provider
36
},
37
};
38
}
39
+function snapshot(id: string, text: string, threadId = "single-thread") {
40
+ const base = event(0, text);
41
+ return base.type === "message.updated"
42
+ ? {
43
+ ...base,
44
+ message: { ...base.message, id: MessageId.make(id), threadId: ThreadId.make(threadId) },
45
+ }
46
+ : base;
47
+}
48
const control: ProviderAdapter.ProviderAdapterV2Event = {
49
type: "background_work.updated",
50
driver,
@@ -45,6 +54,7 @@ const control: ProviderAdapter.ProviderAdapterV2Event = {
54
55
const recordBatches = Effect.fn("test.recordProviderBatches")(function* (
56
beforeCommit: Effect.Effect<void> = Effect.void,
57
+ isViewed?: (threadId: string) => boolean,
58
) {
59
const output = yield* Queue.unbounded<ReadonlyArray<ProviderAdapter.ProviderAdapterV2Event>>();
60
let current: ProviderAdapter.ProviderAdapterV2Event[] = [];
@@ -57,6 +67,8 @@ const recordBatches = Effect.fn("test.recordProviderBatches")(function* (
67
if (current.length > 0) yield* Queue.offer(output, current);
68
}),
69
true,
70
+ undefined,
71
+ isViewed,
72
);
73
const process = (raw: ProviderAdapter.ProviderAdapterV2Event) =>
74
raw.type === "background_work.updated"
@@ -463,6 +475,150 @@ it.effect.each(["none", "writer", "snapshots"] as const)(
475
}),
476
);
477
478
+const blockFirstCommit = Effect.gen(function* () {
479
+ const started = yield* Deferred.make<void>();
480
+ const release = yield* Deferred.make<void>();
481
+ let first = true;
482
+ const block = Effect.gen(function* () {
483
+ if (!first) return;
484
+ first = false;
485
+ yield* Deferred.succeed(started, undefined);
486
+ yield* Deferred.await(release);
487
+ });
488
+ return { started, release, block };
489
+});
490
+
491
+it.effect.each([
492
+ { keys: ["a", "b", "a", "b", "a", "b"], kept: [0, 4, 5] },
493
+ // A first appearance between two snapshots of an entity keeps the earlier snapshot.
494
+ { keys: ["a", "b", "a", "b", "c", "a", "b"], kept: [0, 2, 3, 4, 5, 6] },
495
+])("combines superseded snapshots queued behind a busy commit: $keys", ({ keys, kept }) =>
496
+ Effect.gen(function* () {
497
+ const writer = yield* blockFirstCommit;
498
+ const recorder = yield* recordBatches(writer.block);
499
+ const leading = yield* recorder
500
+ .consume(Stream.make(event(-1)), recorder.process, Effect.succeed(false))
501
+ .pipe(Effect.forkScoped({ startImmediately: true }));
502
+ yield* Deferred.await(writer.started);
503
+ const snapshots = keys.map((key, index) => snapshot(key, `${key}:${index}`));
504
+ const pulled = yield* Deferred.make<void>();
505
+ const consumer = yield* recorder
506
+ .consume(
507
+ Stream.fromIterable(snapshots).pipe(
508
+ Stream.tap((raw) =>
509
+ raw === snapshots.at(-1) ? Deferred.succeed(pulled, undefined) : Effect.void,
510
+ ),
511
+ ),
512
+ recorder.process,
513
+ Effect.succeed(false),
514
+ )
515
+ .pipe(Effect.forkScoped({ startImmediately: true }));
516
+ yield* Deferred.await(pulled);
517
+ yield* TestClock.adjust(0);
518
+ yield* Deferred.succeed(writer.release, undefined);
519
+ yield* Fiber.join(leading);
520
+ yield* Fiber.join(consumer);
521
+ assert.deepEqual(yield* Queue.takeAll(recorder.output), [
522
+ [event(-1)],
523
+ kept.map((index) => snapshots[index]!),
524
+ ]);
525
+ }),
526
+);
527
+
528
+it.effect("makes streams wait to read more once the shared backlog is full", () =>
529
+ Effect.gen(function* () {
530
+ const writer = yield* blockFirstCommit;
531
+ const recorder = yield* recordBatches(writer.block);
532
+ const leading = yield* recorder
533
+ .consume(Stream.make(event(-1)), recorder.process, Effect.succeed(false))
534
+ .pipe(Effect.forkScoped({ startImmediately: true }));
535
+ yield* Deferred.await(writer.started);
536
+ let arrivals = 0;
537
+ const streams = Array.from({ length: 5 }, (_, stream) =>
538
+ Array.from({ length: 64 }, (_, n) => snapshot(`${stream}:${n}`, `${stream}:${n}`)),
539
+ );
540
+ const consumers = yield* Effect.forEach(streams, (snapshots) =>
541
+ recorder
542
+ .consume(
543
+ Stream.fromIterable(snapshots).pipe(
544
+ Stream.tap(() =>
545
+ Effect.sync(() => {
546
+ arrivals++;
547
+ }),
548
+ ),
549
+ ),
550
+ recorder.process,
551
+ Effect.succeed(false),
552
+ )
553
+ .pipe(Effect.forkScoped({ startImmediately: true })),
554
+ );
555
+ yield* TestClock.adjust(0);
556
+ // 256 queued, plus at most one read per stream waiting for room.
557
+ assert.isAtMost(arrivals, 256 + streams.length);
558
+ yield* Deferred.succeed(writer.release, undefined);
559
+ yield* Fiber.join(leading);
560
+ yield* Effect.forEach(consumers, Fiber.join, { discard: true });
561
+ assert.lengthOf((yield* Queue.takeAll(recorder.output)).flat(), 1 + 5 * 64);
562
+ }),
563
+);
564
+
565
+it.effect.each([false, true])(
566
+ "orders a backlog by focus, thread kind and a finishing control with starved=%s",
567
+ (starved) =>
568
+ Effect.gen(function* () {
569
+ const writer = yield* blockFirstCommit;
570
+ const recorder = yield* recordBatches(writer.block, (threadId) => threadId === "focused");
571
+ const leading = yield* recorder
572
+ .consume(Stream.make(event(-1)), recorder.process, Effect.succeed(false))
573
+ .pipe(Effect.forkScoped({ startImmediately: true }));
574
+ yield* Deferred.await(writer.started);
575
+ const streams = [
576
+ { name: "subagent", delegated: true, finishing: false },
577
+ { name: "main", delegated: false, finishing: false },
578
+ { name: "finishing", delegated: false, finishing: true },
579
+ { name: "focused", delegated: false, finishing: false },
580
+ ];
581
+ const pulled = yield* Deferred.make<void>();
582
+ let arrivals = 0;
583
+ const consumers = yield* Effect.forEach(streams, ({ name, delegated, finishing }) =>
584
+ recorder
585
+ .consume(
586
+ // Open streams, so only the control marks a stream as finishing; EOF would too.
587
+ Stream.concat(
588
+ Stream.make(snapshot(name, name, name), ...(finishing ? [control] : [])),
589
+ Stream.never,
590
+ ).pipe(
591
+ Stream.tap(() =>
592
+ ++arrivals === streams.length + 1
593
+ ? Deferred.succeed(pulled, undefined)
594
+ : Effect.void,
595
+ ),
596
+ ),
597
+ recorder.process,
598
+ Effect.succeed(false),
599
+ Effect.succeed(true),
600
+ { threadId: name, delegated },
601
+ )
602
+ .pipe(Effect.forkScoped({ startImmediately: true })),
603
+ );
604
+ yield* Deferred.await(pulled);
605
+ yield* TestClock.adjust(starved ? "30 seconds" : 0);
606
+ yield* Deferred.succeed(writer.release, undefined);
607
+ yield* Fiber.join(leading);
608
+ const outputs = yield* Queue.takeN(recorder.output, 3);
609
+ yield* Effect.forEach(consumers, Fiber.interrupt, { discard: true });
610
+ // Waiting 30 seconds ranks a subagent like a main thread.
611
+ const order = starved
612
+ ? ["focused", "finishing", "subagent", "main"]
613
+ : ["focused", "finishing", "main", "subagent"];
614
+ assert.deepEqual(Array.from(outputs), [
615
+ [event(-1)],
616
+ order.map((name) => snapshot(name, name, name)),
617
+ [control],
618
+ ]);
619
+ }),
620
+);
621
+
622
it.effect.each(["failure", "stop"] as const)(
623
"flushes the prefix and respects a source %s barrier",
624
(mode) =>
apps/server/src/orchestration-v2/ProviderCommitBatch.ts
+249
-96
@@ -7,12 +7,17 @@ import * as Deferred from "effect/Deferred";
7
import * as Exit from "effect/Exit";
8
import * as Queue from "effect/Queue";
9
import * as Schema from "effect/Schema";
10
+import * as Semaphore from "effect/Semaphore";
11
import * as Stream from "effect/Stream";
12
13
const MAX_EVENTS = 64;
14
const MAX_BYTES = 1024 * 1024;
15
// While another write waits, a shared transaction stops taking jobs after this long.
16
const MAX_HOLD_NANOS = 100_000_000n;
17
+// A stream waiting this long ranks like a main thread, so subagents cannot starve.
18
+const STARVATION_MILLIS = 30_000;
19
+// Bounds snapshots held across stream queues; a full backlog makes streams wait to read more.
20
+const MAX_QUEUED = 4 * MAX_EVENTS;
21
const encodeQueue = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown));
22
23
export const enabled = Config.Boolean("T3CODE_PROVIDER_COMMIT_BATCHING").pipe(
@@ -21,35 +26,66 @@ export const enabled = Config.Boolean("T3CODE_PROVIDER_COMMIT_BATCHING").pipe(
26
);
27
28
type Snapshot =
24
- | { readonly type: "node.updated"; readonly node: { readonly threadId: string } }
25
- | { readonly type: "message.updated"; readonly message: { readonly threadId: string } }
26
- | { readonly type: "turn_item.updated"; readonly turnItem: { readonly threadId: string } }
27
- | { readonly type: "plan.updated"; readonly plan: { readonly threadId: string } };
29
+ | {
30
+ readonly type: "node.updated";
31
+ readonly node: { readonly id: string; readonly threadId: string };
32
+ }
33
+ | {
34
+ readonly type: "message.updated";
35
+ readonly message: { readonly id: string; readonly threadId: string };
36
+ }
37
+ | {
38
+ readonly type: "turn_item.updated";
39
+ readonly turnItem: { readonly id: string; readonly threadId: string };
40
+ }
41
+ | {
42
+ readonly type: "plan.updated";
43
+ readonly plan: { readonly id: string; readonly threadId: string };
44
+ };
45
const isSnapshot = (event: { readonly type: string }): event is Snapshot =>
46
event.type === "node.updated" ||
47
event.type === "message.updated" ||
48
event.type === "turn_item.updated" ||
49
event.type === "plan.updated";
33
-const snapshotThreadId = (event: Snapshot) => {
34
- switch (event.type) {
35
- case "node.updated":
36
- return event.node.threadId;
37
- case "message.updated":
38
- return event.message.threadId;
39
- case "turn_item.updated":
40
- return event.turnItem.threadId;
41
- case "plan.updated":
42
- return event.plan.threadId;
43
- }
50
+const snapshotEntity = (event: Snapshot) => {
51
+ const entity =
52
+ event.type === "node.updated"
53
+ ? event.node
54
+ : event.type === "message.updated"
55
+ ? event.message
56
+ : event.type === "turn_item.updated"
57
+ ? event.turnItem
58
+ : event.plan;
59
+ return { key: `${event.type}:${entity.id}`, threadId: entity.threadId };
60
};
61
62
+/** Who a provider stream writes for; ranks its snapshots against other streams'. */
63
+export interface StreamPriority {
64
+ readonly threadId: string;
65
+ /** A subagent or delegated task, which yields to main threads. */
66
+ readonly delegated: boolean;
67
+}
68
+
69
export class CommitBatchError extends Schema.TaggedError<CommitBatchError>()("CommitBatchError", {
70
cause: Schema.Defect(),
71
}) {}
72
export const isCommitBatchError = Schema.is(CommitBatchError);
73
51
-interface Job {
74
+interface StreamState {
75
readonly owner: { active: boolean };
76
+ readonly priority: StreamPriority | undefined;
77
+ /** Queued snapshots not yet taken into a batch, in stream order. */
78
+ readonly jobs: Array<Job>;
79
+ unsettled: number;
80
+ servedAt: number;
81
+ /** A control, failure or EOF of this stream waits for its queued snapshots. */
82
+ finishing: boolean;
83
+}
84
+interface Job {
85
+ readonly stream: StreamState;
86
+ /** Still in its stream's queue, holding one of the backlog's `MAX_QUEUED` slots. */
87
+ queued: boolean;
88
+ readonly key: string;
89
readonly threadId: string;
90
readonly queuedAt: number;
91
readonly bytes: number;
@@ -58,34 +94,51 @@ interface Job {
94
exit: Exit.Exit<void, CommitBatchError>,
95
preserveFailure: boolean,
96
) => Effect.Effect<void>;
97
+ /** Settles a job a later snapshot of the same entity replaced, with that snapshot's outcome. */
98
+ readonly skip: (exit: Exit.Exit<void, CommitBatchError>) => Effect.Effect<void>;
99
readonly cancel: Effect.Effect<void>;
100
readonly settled: Effect.Effect<void>;
101
}
102
+interface Unit {
103
+ readonly job: Job;
104
+ readonly superseded: ReadonlyArray<Job>;
105
+}
106
interface Batch {
65
- readonly events: ReadonlyArray<Job>;
107
+ readonly units: ReadonlyArray<Unit>;
108
readonly reason: "ready" | "limit";
109
+ readonly dequeued: number;
110
}
111
69
-function collect(queue: Queue.Queue<Job>) {
70
- return Stream.fromEffectRepeat(Queue.takeBetween(queue, 1, MAX_EVENTS)).pipe(
71
- Stream.flatMap((ready) => {
72
- const batches: Array<Batch> = [];
73
- let events: Array<Job> = [];
74
- let bytes = 0;
75
- const flush = (reason: Batch["reason"]) => {
76
- if (events.length > 0) batches.push({ events, reason });
77
- events = [];
78
- bytes = 0;
79
- };
80
- for (const job of ready) {
81
- if (events.length > 0 && bytes + job.bytes > MAX_BYTES) flush("limit");
82
- events.push(job);
83
- bytes += job.bytes;
84
- if (events.length >= MAX_EVENTS || bytes >= MAX_BYTES) flush("limit");
85
- }
86
- flush("ready");
87
- return Stream.fromArray(batches);
88
- }),
112
+/**
113
+ * Snapshots carry an entity's full state, so an earlier one is dropped when a later one of the
114
+ * same entity follows it in this stream with no first appearance of another entity in between.
115
+ * Kept jobs stay in stream order, and entities first appear in the same order as before.
116
+ */
117
+function coalesce(jobs: ReadonlyArray<Job>): Array<Unit> {
118
+ const firstSeen = new Map<string, number>();
119
+ jobs.forEach((job, index) => {
120
+ if (!firstSeen.has(job.key)) firstSeen.set(job.key, index);
121
+ });
122
+ const survivor: Array<number> = Array.from({ length: jobs.length }, (_, index) => index);
123
+ const nextOfKey = new Map<string, number>();
124
+ for (let index = jobs.length - 1; index >= 0; index--) {
125
+ const job = jobs[index]!;
126
+ const next = nextOfKey.get(job.key);
127
+ nextOfKey.set(job.key, index);
128
+ if (next === undefined) continue;
129
+ let replaced = true;
130
+ for (let between = index + 1; replaced && between < next; between++) {
131
+ if (firstSeen.get(jobs[between]!.key)! > index) replaced = false;
132
+ }
133
+ if (replaced) survivor[index] = survivor[next]!;
134
+ }
135
+ const superseded = new Map<number, Array<Job>>();
136
+ jobs.forEach((job, index) => {
137
+ const kept = survivor[index]!;
138
+ if (kept !== index) superseded.set(kept, [...(superseded.get(kept) ?? []), job]);
139
+ });
140
+ return jobs.flatMap((job, index) =>
141
+ survivor[index] === index ? [{ job, superseded: superseded.get(index) ?? [] }] : [],
142
);
143
}
144
@@ -102,32 +155,44 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* (
155
isolate: (
156
work: Effect.Effect<void, CommitBatchError>,
157
) => Effect.Effect<void, CommitBatchError> = (work) => work,
158
+ /** Whether a client has the thread open; its streams rank first. */
159
+ isViewed: (threadId: string) => boolean = () => false,
160
) {
106
- const queue = yield* Queue.bounded<Job>(MAX_EVENTS);
161
+ const wake = yield* Queue.sliding<void>(1);
162
+ const backlog = yield* Semaphore.make(MAX_QUEUED);
163
+ const streams = new Set<StreamState>();
164
const pending = new Set<Job>();
165
let closed = false;
166
+ let servings = 0;
167
// Writes in `barrier` that have not finished; they wait on the batch or the writer it holds.
168
let waitingWriters = 0;
169
const shutdown = Effect.gen(function* () {
170
closed = true;
113
- yield* Queue.shutdown(queue);
171
+ yield* Queue.shutdown(wake);
172
for (const job of pending) yield* job.cancel;
173
+ yield* backlog.releaseAll;
174
});
175
yield* Effect.addFinalizer(() => shutdown);
176
// These helpers run once per raw event; the shared transaction owns the tracing span.
177
const enqueue = Effect.fnUntraced(function* <A, E, R>(
178
effect: Effect.Effect<A, E, R>,
120
- token: { active: boolean },
121
- threadId: string,
179
+ stream: StreamState,
180
+ entity: { readonly key: string; readonly threadId: string },
181
bytes: number,
182
+ skipped: A,
183
) {
184
const caller = yield* Effect.context<R>();
185
const reply = yield* Deferred.make<A, E | CommitBatchError>();
186
let ownFailure: Cause.Cause<E> | undefined;
187
let result: A;
188
+ const release = () => {
189
+ if (pending.delete(job)) stream.unsettled--;
190
+ };
191
const job: Job = {
129
- owner: token,
130
- threadId,
192
+ stream,
193
+ queued: false,
194
+ key: entity.key,
195
+ threadId: entity.threadId,
196
queuedAt: yield* Clock.currentTimeMillis,
197
bytes,
198
execute: effect.pipe(
@@ -149,44 +214,111 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* (
214
exit: Exit.Exit<void, CommitBatchError>,
215
preserveFailure: boolean,
216
) {
152
- pending.delete(job);
217
+ release();
218
if (Exit.isFailure(exit)) {
154
- token.active = false;
219
+ stream.owner.active = false;
220
yield* Deferred.failCause(
221
reply,
222
preserveFailure && ownFailure !== undefined ? ownFailure : exit.cause,
223
);
224
} else yield* Deferred.succeed(reply, result);
225
}),
161
- cancel: Effect.gen(function* () {
162
- pending.delete(job);
163
- yield* Deferred.interrupt(reply);
226
+ skip: (exit) =>
227
+ Exit.isFailure(exit)
228
+ ? job.settle(exit, false)
229
+ : Effect.suspend(() => {
230
+ release();
231
+ return Deferred.succeed(reply, skipped);
232
+ }),
233
+ cancel: Effect.suspend(() => {
234
+ release();
235
+ const dequeued = job.queued;
236
+ job.queued = false;
237
+ return Deferred.interrupt(reply).pipe(
238
+ Effect.andThen(dequeued ? backlog.release(1) : Effect.void),
239
+ );
240
}),
241
settled: Deferred.await(reply).pipe(Effect.exit, Effect.asVoid),
242
};
243
yield* Effect.uninterruptibleMask((restore) =>
244
Effect.gen(function* () {
245
+ if (closed) return yield* Deferred.interrupt(reply);
246
+ yield* restore(backlog.take(1));
247
+ if (closed) {
248
+ yield* backlog.release(1);
249
+ return yield* Deferred.interrupt(reply);
250
+ }
251
+ job.queued = true;
252
pending.add(job);
170
- const accepted = yield* restore(Queue.offer(queue, job)).pipe(
171
- Effect.onError(() => job.cancel),
172
- );
173
- if (!accepted) yield* job.cancel;
253
+ stream.unsettled++;
254
+ stream.jobs.push(job);
255
+ yield* Queue.offer(wake, undefined);
256
}),
257
);
258
return Deferred.await(reply);
259
});
178
- // Commits a prefix of `jobs` in one transaction and returns how many it took.
179
- const commitPrefix = (jobs: ReadonlyArray<Job>, reason: Batch["reason"]) =>
260
+ const rank = (stream: StreamState, now: number) => {
261
+ const tier =
262
+ stream.priority === undefined
263
+ ? 1
264
+ : isViewed(stream.priority.threadId)
265
+ ? 0
266
+ : stream.priority.delegated
267
+ ? 2
268
+ : 1;
269
+ const starved = now - stream.jobs[0]!.queuedAt >= STARVATION_MILLIS;
270
+ return [starved ? Math.min(tier, 1) : tier, stream.finishing ? 0 : 1, stream.servedAt];
271
+ };
272
+ // Viewed streams first, then main threads, then subagents; within a tier, streams finishing a
273
+ // control first, then the stream served longest ago.
274
+ const nextBatch = (now: number): Batch | undefined => {
275
+ const ready = Array.from(streams, (stream) =>
276
+ stream.jobs.length === 0 ? [] : [{ stream, rank: rank(stream, now) }],
277
+ )
278
+ .flat()
279
+ .sort((left, right) => {
280
+ for (let index = 0; index < left.rank.length; index++) {
281
+ const order = left.rank[index]! - right.rank[index]!;
282
+ if (order !== 0) return order;
283
+ }
284
+ return 0;
285
+ });
286
+ if (ready.length === 0) return undefined;
287
+ const units: Array<Unit> = [];
288
+ let bytes = 0;
289
+ let dequeued = 0;
290
+ let reason: Batch["reason"] = "ready";
291
+ for (const { stream } of ready) {
292
+ let take = 0;
293
+ while (take < stream.jobs.length && take < MAX_EVENTS - units.length) {
294
+ const next = stream.jobs[take]!.bytes;
295
+ if (units.length + take > 0 && bytes + next > MAX_BYTES) break;
296
+ bytes += next;
297
+ take++;
298
+ }
299
+ if (take < stream.jobs.length) reason = "limit";
300
+ if (take === 0) break;
301
+ const taken = stream.jobs.splice(0, take);
302
+ for (const job of taken) job.queued = false;
303
+ dequeued += take;
304
+ units.push(...coalesce(taken));
305
+ stream.servedAt = ++servings;
306
+ if (units.length >= MAX_EVENTS || bytes >= MAX_BYTES) break;
307
+ }
308
+ return { units, reason, dequeued };
309
+ };
310
+ // Commits a prefix of `units` in one transaction and returns how many it took.
311
+ const commitPrefix = (units: ReadonlyArray<Unit>, reason: Batch["reason"]) =>
312
Effect.uninterruptibleMask((restore) =>
313
Effect.gen(function* () {
314
const outcomes = new Map<Job, Exit.Exit<void, CommitBatchError>>();
315
let taken = 0;
316
const work = Effect.gen(function* () {
185
- // A retried transaction runs this again from the first job.
317
+ // A retried transaction runs this again from the first unit.
318
outcomes.clear();
319
taken = 0;
320
const startedAt = yield* Clock.monotonicTimeNanos;
189
- for (const job of jobs) {
321
+ for (const { job, superseded } of units) {
322
if (
323
taken > 0 &&
324
waitingWriters > 0 &&
@@ -194,63 +326,62 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* (
326
)
327
return;
328
taken++;
197
- if (!job.owner.active) {
329
+ if (!job.stream.owner.active) {
330
yield* job.cancel;
331
+ for (const replaced of superseded) yield* replaced.cancel;
332
continue;
333
}
201
- const outcome = yield* (jobs.length > 1 ? isolate(job.execute) : job.execute).pipe(
334
+ const outcome = yield* (units.length > 1 ? isolate(job.execute) : job.execute).pipe(
335
Effect.exit,
336
);
337
outcomes.set(job, outcome);
338
if (Exit.isFailure(outcome)) {
206
- job.owner.active = false;
207
- if (jobs.length === 1) return yield* Effect.failCause(outcome.cause);
339
+ job.stream.owner.active = false;
340
+ if (units.length === 1) return yield* Effect.failCause(outcome.cause);
341
}
342
}
343
});
344
const exit = yield* restore(
345
commit(work).pipe(Effect.provideService(CurrentWorker, yield* Effect.fiberId)),
346
).pipe(Effect.exit);
214
- const committed = taken === 0 ? jobs : jobs.slice(0, taken);
347
+ const committed = taken === 0 ? units : units.slice(0, taken);
348
+ const jobs = committed.flatMap((unit) => [unit.job, ...unit.superseded]);
349
yield* Effect.annotateCurrentSpan({
350
"orchestration_v2.work_item_count": committed.length,
217
- "orchestration_v2.deferred_work_item_count": jobs.length - committed.length,
218
- "orchestration_v2.consumer_count": new Set(committed.map((job) => job.owner)).size,
219
- "orchestration_v2.raw_json_bytes": committed.reduce((bytes, job) => bytes + job.bytes, 0),
351
+ "orchestration_v2.coalesced_work_item_count": jobs.length - committed.length,
352
+ "orchestration_v2.deferred_work_item_count": units.length - committed.length,
353
+ "orchestration_v2.consumer_count": new Set(jobs.map((job) => job.stream)).size,
354
+ "orchestration_v2.raw_json_bytes": jobs.reduce((bytes, job) => bytes + job.bytes, 0),
355
"orchestration_v2.batch_flush_reason": reason,
356
"orchestration_v2.backlog_batching": true,
357
"orchestration_v2.batch_wait_ms":
223
- (yield* Clock.currentTimeMillis) - Math.min(...committed.map((job) => job.queuedAt)),
358
+ (yield* Clock.currentTimeMillis) - Math.min(...jobs.map((job) => job.queuedAt)),
359
});
225
- for (const job of committed) {
360
+ for (const { job, superseded } of committed) {
361
const outcome = outcomes.get(job) ?? Exit.void;
362
+ const final = Exit.isSuccess(exit) ? outcome : exit;
363
yield* job.settle(
228
- Exit.isSuccess(exit) ? outcome : exit,
364
+ final,
365
Exit.isSuccess(exit) ||
366
(Exit.isFailure(exit) &&
367
Exit.isFailure(outcome) &&
368
Cause.squash(exit.cause) === Cause.squash(outcome.cause)),
369
);
370
+ for (const replaced of superseded) yield* replaced.skip(final);
371
}
372
return committed.length;
373
}),
374
).pipe(Effect.withSpan("orchestrationV2.ProviderCommitBatch.flush"));
375
if (enabled) {
239
- yield* collect(queue).pipe(
240
- Stream.runForEach(
241
- Effect.fnUntraced(function* (batch: Batch) {
242
- let jobs = batch.events;
243
- while (jobs.length > 0) {
244
- for (const job of jobs) if (!job.owner.active) yield* job.cancel;
245
- jobs = jobs.filter((job) => job.owner.active);
246
- if (jobs.length === 0) return;
247
- jobs = jobs.slice(yield* commitPrefix(jobs, batch.reason));
248
- }
249
- }),
250
- ),
251
- Effect.ensuring(shutdown),
252
- Effect.forkScoped,
253
- );
376
+ yield* Effect.forever(
377
+ Effect.gen(function* () {
378
+ const batch = nextBatch(yield* Clock.currentTimeMillis);
379
+ if (batch === undefined) return yield* Queue.take(wake);
380
+ yield* backlog.release(batch.dequeued);
381
+ let units = batch.units;
382
+ while (units.length > 0) units = units.slice(yield* commitPrefix(units, batch.reason));
383
+ }),
384
+ ).pipe(Effect.ensuring(shutdown), Effect.forkScoped);
385
}
386
/**
387
* Runs `effect` after queued snapshots for `threadIds` commit; omitted, after every queued
@@ -286,6 +417,7 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* (
417
process: (event: A) => Effect.Effect<void, E2, R2>,
418
after: Effect.Effect<boolean, E2, R2>,
419
canBatch: Effect.Effect<boolean, never, R2> = Effect.succeed(true),
420
+ priority?: StreamPriority,
421
) {
422
if (!enabled)
423
return yield* source.pipe(
@@ -293,7 +425,15 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* (
425
Stream.takeUntil(Boolean),
426
Stream.runDrain,
427
);
296
- const token = { active: true };
428
+ const stream: StreamState = {
429
+ owner: { active: true },
430
+ priority,
431
+ jobs: [],
432
+ unsettled: 0,
433
+ servedAt: 0,
434
+ finishing: false,
435
+ };
436
+ streams.add(stream);
437
const signals = source.pipe(
438
Stream.map((event) => ({ _tag: "Event" as const, event })),
439
Stream.catchCause((cause) => Stream.succeed({ _tag: "Failure" as const, cause })),
@@ -309,20 +449,31 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* (
449
const processed = yield* Deferred.make<void>();
450
let reply: Effect.Effect<boolean, E | E2 | CommitBatchError, R2>;
451
let check = false;
312
- if (signal._tag === "End") reply = Effect.succeed(true);
313
- else if (signal._tag === "Failure") reply = Effect.failCause(signal.cause);
314
- else if (isSnapshot(signal.event) && (yield* canBatch)) {
315
- const bytes = Buffer.byteLength(yield* encodeQueue(signal.event));
452
+ const snapshot =
453
+ signal._tag === "Event" && isSnapshot(signal.event) && (yield* canBatch)
454
+ ? signal.event
455
+ : undefined;
456
+ if (snapshot !== undefined) {
457
+ const bytes = Buffer.byteLength(yield* encodeQueue(snapshot));
458
reply = yield* enqueue(
317
- process(signal.event).pipe(Effect.as(false)),
318
- token,
319
- snapshotThreadId(signal.event),
459
+ process(snapshot).pipe(Effect.as(false)),
460
+ stream,
461
+ snapshotEntity(snapshot),
462
bytes,
463
+ false,
464
);
465
} else {
323
- reply = process(signal.event).pipe(Effect.as(false));
324
- check = true;
325
- resume = Deferred.await(processed);
466
+ if (stream.unsettled > 0) stream.finishing = true;
467
+ if (signal._tag === "End") reply = Effect.succeed(true);
468
+ else if (signal._tag === "Failure") reply = Effect.failCause(signal.cause);
469
+ else {
470
+ const event = signal.event;
471
+ reply = Effect.sync(() => {
472
+ stream.finishing = false;
473
+ }).pipe(Effect.andThen(process(event)), Effect.as(false));
474
+ check = true;
475
+ resume = Deferred.await(processed);
476
+ }
477
}
478
return { reply, processed, check };
479
}),
@@ -335,7 +486,7 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* (
486
),
487
Effect.tap((stop) =>
488
Effect.sync(() => {
338
- if (stop) token.active = false;
489
+ if (stop) stream.owner.active = false;
490
}),
491
),
492
Effect.ensuring(Deferred.succeed(processed, undefined)),
@@ -344,8 +495,10 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* (
495
Stream.takeUntil(Boolean),
496
Stream.runDrain,
497
Effect.ensuring(
347
- Effect.sync(() => {
348
- token.active = false;
498
+ Effect.gen(function* () {
499
+ stream.owner.active = false;
500
+ streams.delete(stream);
501
+ for (const job of stream.jobs.splice(0)) yield* job.cancel;
502
}),
503
),
504
);
apps/server/src/orchestration-v2/RunExecutionService.ts
+2
@@ -1,6 +1,7 @@
1
import { makeAssistantStreamingFilter } from "./assistantStreaming.ts";
2
import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
3
import {
4
+ isDelegatedAppThread,
5
isOrchestrationV2WorkActive,
6
CommandId,
7
type EventId,
@@ -1988,6 +1989,7 @@ export const layer: Layer.Layer<
1989
return yield* shouldStopProviderEventIngestion;
1990
}),
1991
Ref.get(rootTerminalSeen).pipe(Effect.map((seen) => !seen)),
1992
+ { threadId: input.run.threadId, delegated: isDelegatedAppThread(input.appThread) },
1993
)
1994
.pipe(
1995
Effect.ensuring(closeIdleFlush),
apps/server/src/orchestration-v2/ThreadViewers.ts
new
+33
@@ -0,0 +1,33 @@
1
+import * as Context from "effect/Context";
2
+import * as Effect from "effect/Effect";
3
+import * as Layer from "effect/Layer";
4
+import type * as Scope from "effect/Scope";
5
+
6
+export class ThreadViewers extends Context.Service<
7
+ ThreadViewers,
8
+ {
9
+ /** Counts a client viewing `threadId` until the enclosing scope closes. */
10
+ readonly view: (threadId: string) => Effect.Effect<void, never, Scope.Scope>;
11
+ readonly isViewed: (threadId: string) => boolean;
12
+ }
13
+>()("t3/orchestration-v2/ThreadViewers") {}
14
+
15
+export const layer = Layer.sync(ThreadViewers, () => {
16
+ const viewers = new Map<string, number>();
17
+ return ThreadViewers.of({
18
+ view: Effect.fn("ThreadViewers.view")(function* (threadId: string) {
19
+ yield* Effect.acquireRelease(
20
+ Effect.sync(() => {
21
+ viewers.set(threadId, (viewers.get(threadId) ?? 0) + 1);
22
+ }),
23
+ () =>
24
+ Effect.sync(() => {
25
+ const remaining = (viewers.get(threadId) ?? 1) - 1;
26
+ if (remaining > 0) viewers.set(threadId, remaining);
27
+ else viewers.delete(threadId);
28
+ }),
29
+ );
30
+ }),
31
+ isViewed: (threadId) => viewers.has(threadId),
32
+ });
33
+});
apps/server/src/orchestration-v2/runtimeLayer.ts
+5
-1
@@ -28,6 +28,7 @@ import * as ContextHandoffService from "./ContextHandoffService.ts";
28
import * as EffectOutbox from "./EffectOutbox.ts";
29
import * as EffectWorker from "./EffectWorker.ts";
30
import * as EventSink from "./EventSink.ts";
31
+import * as ThreadViewers from "./ThreadViewers.ts";
32
import * as EventStore from "./EventStore.ts";
33
import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
34
import * as LegacyV1ThreadImporter from "./legacy/LegacyV1ThreadImporter.ts";
@@ -86,7 +87,10 @@ const layerStores = Layer.mergeAll(
87
TurnItemPositionStore.layer,
88
);
89
89
-export const layerEventSink = EventSink.layerFromStores.pipe(Layer.provide(layerStores));
90
+// The server provides this same layer to thread subscriptions; memoized, they share one registry.
91
+export const layerEventSink = EventSink.layerFromStores.pipe(
92
+ Layer.provide(Layer.merge(layerStores, ThreadViewers.layer)),
93
+);
94
const layerProjectionMaintenanceProvided = ProjectionMaintenance.layer.pipe(
95
Layer.provide(layerStores),
96
);
apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts
+2
-1
@@ -37,6 +37,7 @@ import * as ContextHandoffService from "../ContextHandoffService.ts";
37
import * as EffectOutbox from "../EffectOutbox.ts";
38
import * as EffectWorker from "../EffectWorker.ts";
39
import * as EventSink from "../EventSink.ts";
40
+import * as ThreadViewers from "../ThreadViewers.ts";
41
import * as EventStore from "../EventStore.ts";
42
import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
43
import * as Orchestrator from "../Orchestrator.ts";
@@ -349,7 +350,7 @@ export function layerWithRegistry<Error>(
350
TurnItemPositionStore.layer,
351
).pipe(Layer.provide(layerDatabase));
352
const layerEventSinkProvided = EventSink.layerFromStores.pipe(
352
- Layer.provide(Layer.mergeAll(layerStores, layerDatabase)),
353
+ Layer.provide(Layer.mergeAll(layerStores, layerDatabase, ThreadViewers.layer)),
354
);
355
const layerRuntime = RuntimePolicy.layerEnforcingCeiling.pipe(
356
Layer.provide(Layer.merge(layerRuntimeBase, layerServerSettings)),
apps/server/src/server.ts
+3
@@ -215,6 +215,7 @@ import * as CursorUsageAccounts from "@t3tools/provider-cursor/server/CursorUsag
215
import * as UsageService from "./usage/UsageService.ts";
216
import * as CodexUsageHistory from "./usage/CodexUsageHistory.ts";
217
import * as RuntimeLayer from "./orchestration-v2/runtimeLayer.ts";
218
+import * as ThreadViewers from "./orchestration-v2/ThreadViewers.ts";
219
import * as ProjectStore from "./orchestration-v2/ProjectStore.ts";
220
import * as ThreadSearch from "./orchestration-v2/ThreadSearch.ts";
221
import * as ThreadSettlementService from "./orchestration-v2/ThreadSettlementService.ts";
@@ -814,6 +815,8 @@ const layerRuntimeCoreDependencies = layerRuntimeCoreDependenciesBase.pipe(
815
const layerRuntimeDependencies = layerRuntimeCoreDependencies.pipe(
816
Layer.provideMerge(layerUsage),
817
Layer.provideMerge(layerBackground),
818
+ // The same layer the event sink uses; memoized, subscriptions and batch ranking share it.
819
+ Layer.provideMerge(ThreadViewers.layer),
820
Layer.provideMerge(layerResourceDiagnostics),
821
Layer.provideMerge(TraceDiagnostics.layer),
822
Layer.provideMerge(TriageService.layer.pipe(Layer.provide(GitVcsDriver.layer))),
apps/server/src/ws.test.ts
+2
@@ -82,6 +82,7 @@ import * as BackgroundPolicy from "./background/BackgroundPolicy.ts";
82
import * as CheckpointDiffQuery from "./checkpointing/CheckpointDiffQuery.ts";
83
import * as NativeChildWorkQuery from "./orchestration-v2/NativeChildWorkQuery.ts";
84
import * as CommandOutputHub from "./orchestration-v2/CommandOutputHub.ts";
85
+import * as ThreadViewers from "./orchestration-v2/ThreadViewers.ts";
86
import * as CodexInstallation from "./provider/CodexInstallation.ts";
87
import * as DeviceService from "./device/DeviceService.ts";
88
import * as DocumentText from "./provider/DocumentText.ts";
@@ -832,6 +833,7 @@ const layerUnusedConfigRpcServices = Layer.mergeAll(
833
Layer.mock(CheckpointDiffQuery.CheckpointDiffQuery)({}),
834
Layer.mock(NativeChildWorkQuery.NativeChildWorkQuery)({}),
835
Layer.mock(CommandOutputHub.CommandOutputHub)({}),
836
+ ThreadViewers.layer,
837
Layer.mock(CodexInstallation.CodexInstallation)({ managedDirectory: "/unused" }),
838
Layer.mock(DeviceService.DeviceService)({}),
839
Layer.mock(EnvironmentTheme.EnvironmentThemeService)({}),
apps/server/src/ws.ts
+3
@@ -119,6 +119,7 @@ import * as EnvironmentTheme from "./environmentTheme.ts";
119
import * as Keybindings from "./keybindings.ts";
120
import * as ExternalLauncher from "./process/externalLauncher.ts";
121
import * as ThreadManagementService from "./orchestration-v2/ThreadManagementService.ts";
122
+import * as ThreadViewers from "./orchestration-v2/ThreadViewers.ts";
123
import * as CommandOutputHub from "./orchestration-v2/CommandOutputHub.ts";
124
import * as McpAppRequests from "./mcpApps/McpAppRequests.ts";
125
import * as ProviderSessionManager from "./orchestration-v2/ProviderSessionManager.ts";
@@ -1480,6 +1481,7 @@ export const layerWsRpc = (
1481
};
1482
const threadLaunch = yield* ThreadLaunchService.ThreadLaunchService;
1483
const commandOutputHub = yield* CommandOutputHub.CommandOutputHub;
1484
+ const threadViewers = yield* ThreadViewers.ThreadViewers;
1485
const providerSessionManager = yield* ProviderSessionManager.ProviderSessionManagerV2;
1486
const scheduledTasks = yield* ScheduledTasks.ScheduledTaskService;
1487
const secretRequests = yield* SecretRequests.SecretRequests;
@@ -2322,6 +2324,7 @@ export const layerWsRpc = (
2324
[ORCHESTRATION_V2_WS_METHODS.subscribeThread]: (input) =>
2325
Stream.unwrap(
2326
Effect.annotateCurrentSpan({ "orchestration_v2.thread_id": input.threadId }).pipe(
2327
+ Effect.andThen(threadViewers.view(input.threadId)),
2328
Effect.andThen(subscribeOrchestrationV2Thread(input)),
2329
),
2330
),