The best way to control Claude Code, Codex and more from any computer or phone

main
ts 507 lines 18.5 KB
Raw
1 import * as Cause from "effect/Cause";
2 import * as Clock from "effect/Clock";
3 import * as Config from "effect/Config";
4 import * as Effect from "effect/Effect";
5 import * as Context from "effect/Context";
6 import * as Deferred from "effect/Deferred";
7 import * as Exit from "effect/Exit";
8 import * as Queue from "effect/Queue";
9 import * as Schema from "effect/Schema";
10 import * as Semaphore from "effect/Semaphore";
11 import * as Stream from "effect/Stream";
12
13 const MAX_EVENTS = 64;
14 const MAX_BYTES = 1024 * 1024;
15 // While another write waits, a shared transaction stops taking jobs after this long.
16 const MAX_HOLD_NANOS = 100_000_000n;
17 // A stream waiting this long ranks like a main thread, so subagents cannot starve.
18 const STARVATION_MILLIS = 30_000;
19 // Bounds snapshots held across stream queues; a full backlog makes streams wait to read more.
20 const MAX_QUEUED = 4 * MAX_EVENTS;
21 const encodeQueue = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown));
22
23 export const enabled = Config.Boolean("T3CODE_PROVIDER_COMMIT_BATCHING").pipe(
24 Config.withDefault(true),
25 Effect.orDie,
26 );
27
28 type Snapshot =
29 | {
30 readonly type: "node.updated";
31 readonly node: { readonly id: string; readonly threadId: string };
32 }
33 | {
34 readonly type: "message.updated";
35 readonly message: { readonly id: string; readonly threadId: string };
36 }
37 | {
38 readonly type: "turn_item.updated";
39 readonly turnItem: { readonly id: string; readonly threadId: string };
40 }
41 | {
42 readonly type: "plan.updated";
43 readonly plan: { readonly id: string; readonly threadId: string };
44 };
45 const isSnapshot = (event: { readonly type: string }): event is Snapshot =>
46 event.type === "node.updated" ||
47 event.type === "message.updated" ||
48 event.type === "turn_item.updated" ||
49 event.type === "plan.updated";
50 const snapshotEntity = (event: Snapshot) => {
51 const entity =
52 event.type === "node.updated"
53 ? event.node
54 : event.type === "message.updated"
55 ? event.message
56 : event.type === "turn_item.updated"
57 ? event.turnItem
58 : event.plan;
59 return { key: `${event.type}:${entity.id}`, threadId: entity.threadId };
60 };
61
62 /** Who a provider stream writes for; ranks its snapshots against other streams'. */
63 export interface StreamPriority {
64 readonly threadId: string;
65 /** A subagent or delegated task, which yields to main threads. */
66 readonly delegated: boolean;
67 }
68
69 export class CommitBatchError extends Schema.TaggedError<CommitBatchError>()("CommitBatchError", {
70 cause: Schema.Defect(),
71 }) {}
72 export const isCommitBatchError = Schema.is(CommitBatchError);
73
74 interface StreamState {
75 readonly owner: { active: boolean };
76 readonly priority: StreamPriority | undefined;
77 /** Queued snapshots not yet taken into a batch, in stream order. */
78 readonly jobs: Array<Job>;
79 unsettled: number;
80 servedAt: number;
81 /** A control, failure or EOF of this stream waits for its queued snapshots. */
82 finishing: boolean;
83 }
84 interface Job {
85 readonly stream: StreamState;
86 /** Still in its stream's queue, holding one of the backlog's `MAX_QUEUED` slots. */
87 queued: boolean;
88 readonly key: string;
89 readonly threadId: string;
90 readonly queuedAt: number;
91 readonly bytes: number;
92 readonly execute: Effect.Effect<void, CommitBatchError>;
93 readonly settle: (
94 exit: Exit.Exit<void, CommitBatchError>,
95 preserveFailure: boolean,
96 ) => Effect.Effect<void>;
97 /** Settles a job a later snapshot of the same entity replaced, with that snapshot's outcome. */
98 readonly skip: (exit: Exit.Exit<void, CommitBatchError>) => Effect.Effect<void>;
99 readonly cancel: Effect.Effect<void>;
100 readonly settled: Effect.Effect<void>;
101 }
102 interface Unit {
103 readonly job: Job;
104 readonly superseded: ReadonlyArray<Job>;
105 }
106 interface Batch {
107 readonly units: ReadonlyArray<Unit>;
108 readonly reason: "ready" | "limit";
109 readonly dequeued: number;
110 }
111
112 /**
113 * Snapshots carry an entity's full state, so an earlier one is dropped when a later one of the
114 * same entity follows it in this stream with no first appearance of another entity in between.
115 * Kept jobs stay in stream order, and entities first appear in the same order as before.
116 */
117 function coalesce(jobs: ReadonlyArray<Job>): Array<Unit> {
118 const firstSeen = new Map<string, number>();
119 jobs.forEach((job, index) => {
120 if (!firstSeen.has(job.key)) firstSeen.set(job.key, index);
121 });
122 const survivor: Array<number> = Array.from({ length: jobs.length }, (_, index) => index);
123 const nextOfKey = new Map<string, number>();
124 for (let index = jobs.length - 1; index >= 0; index--) {
125 const job = jobs[index]!;
126 const next = nextOfKey.get(job.key);
127 nextOfKey.set(job.key, index);
128 if (next === undefined) continue;
129 let replaced = true;
130 for (let between = index + 1; replaced && between < next; between++) {
131 if (firstSeen.get(jobs[between]!.key)! > index) replaced = false;
132 }
133 if (replaced) survivor[index] = survivor[next]!;
134 }
135 const superseded = new Map<number, Array<Job>>();
136 jobs.forEach((job, index) => {
137 const kept = survivor[index]!;
138 if (kept !== index) superseded.set(kept, [...(superseded.get(kept) ?? []), job]);
139 });
140 return jobs.flatMap((job, index) =>
141 survivor[index] === index ? [{ job, superseded: superseded.get(index) ?? [] }] : [],
142 );
143 }
144
145 const CurrentWorker = Context.Reference<number | undefined>(
146 "t3/ProviderCommitBatch/CurrentWorker",
147 {
148 defaultValue: () => undefined,
149 },
150 );
151
152 export const make = Effect.fn("ProviderCommitBatch.make")(function* (
153 commit: (work: Effect.Effect<void, CommitBatchError>) => Effect.Effect<void, CommitBatchError>,
154 enabled: boolean,
155 isolate: (
156 work: Effect.Effect<void, CommitBatchError>,
157 ) => Effect.Effect<void, CommitBatchError> = (work) => work,
158 /** Whether a client has the thread open; its streams rank first. */
159 isViewed: (threadId: string) => boolean = () => false,
160 ) {
161 const wake = yield* Queue.sliding<void>(1);
162 const backlog = yield* Semaphore.make(MAX_QUEUED);
163 const streams = new Set<StreamState>();
164 const pending = new Set<Job>();
165 let closed = false;
166 let servings = 0;
167 // Writes in `barrier` that have not finished; they wait on the batch or the writer it holds.
168 let waitingWriters = 0;
169 const shutdown = Effect.gen(function* () {
170 closed = true;
171 yield* Queue.shutdown(wake);
172 for (const job of pending) yield* job.cancel;
173 yield* backlog.releaseAll;
174 });
175 yield* Effect.addFinalizer(() => shutdown);
176 // These helpers run once per raw event; the shared transaction owns the tracing span.
177 const enqueue = Effect.fnUntraced(function* <A, E, R>(
178 effect: Effect.Effect<A, E, R>,
179 stream: StreamState,
180 entity: { readonly key: string; readonly threadId: string },
181 bytes: number,
182 skipped: A,
183 ) {
184 const caller = yield* Effect.context<R>();
185 const reply = yield* Deferred.make<A, E | CommitBatchError>();
186 let ownFailure: Cause.Cause<E> | undefined;
187 let result: A;
188 const release = () => {
189 if (pending.delete(job)) stream.unsettled--;
190 };
191 const job: Job = {
192 stream,
193 queued: false,
194 key: entity.key,
195 threadId: entity.threadId,
196 queuedAt: yield* Clock.currentTimeMillis,
197 bytes,
198 execute: effect.pipe(
199 Effect.tapCause((cause) =>
200 Effect.sync(() => {
201 ownFailure = cause;
202 }),
203 ),
204 Effect.mapError((cause) => new CommitBatchError({ cause })),
205 Effect.provideContext(caller),
206 Effect.tap((value) =>
207 Effect.sync(() => {
208 result = value;
209 }),
210 ),
211 Effect.asVoid,
212 ),
213 settle: Effect.fnUntraced(function* (
214 exit: Exit.Exit<void, CommitBatchError>,
215 preserveFailure: boolean,
216 ) {
217 release();
218 if (Exit.isFailure(exit)) {
219 stream.owner.active = false;
220 yield* Deferred.failCause(
221 reply,
222 preserveFailure && ownFailure !== undefined ? ownFailure : exit.cause,
223 );
224 } else yield* Deferred.succeed(reply, result);
225 }),
226 skip: (exit) =>
227 Exit.isFailure(exit)
228 ? job.settle(exit, false)
229 : Effect.suspend(() => {
230 release();
231 return Deferred.succeed(reply, skipped);
232 }),
233 cancel: Effect.suspend(() => {
234 release();
235 const dequeued = job.queued;
236 job.queued = false;
237 return Deferred.interrupt(reply).pipe(
238 Effect.andThen(dequeued ? backlog.release(1) : Effect.void),
239 );
240 }),
241 settled: Deferred.await(reply).pipe(Effect.exit, Effect.asVoid),
242 };
243 yield* Effect.uninterruptibleMask((restore) =>
244 Effect.gen(function* () {
245 if (closed) return yield* Deferred.interrupt(reply);
246 yield* restore(backlog.take(1));
247 if (closed) {
248 yield* backlog.release(1);
249 return yield* Deferred.interrupt(reply);
250 }
251 job.queued = true;
252 pending.add(job);
253 stream.unsettled++;
254 stream.jobs.push(job);
255 yield* Queue.offer(wake, undefined);
256 }),
257 );
258 return Deferred.await(reply);
259 });
260 const rank = (stream: StreamState, now: number) => {
261 const tier =
262 stream.priority === undefined
263 ? 1
264 : isViewed(stream.priority.threadId)
265 ? 0
266 : stream.priority.delegated
267 ? 2
268 : 1;
269 const starved = now - stream.jobs[0]!.queuedAt >= STARVATION_MILLIS;
270 return [starved ? Math.min(tier, 1) : tier, stream.finishing ? 0 : 1, stream.servedAt];
271 };
272 // Viewed streams first, then main threads, then subagents; within a tier, streams finishing a
273 // control first, then the stream served longest ago.
274 const nextBatch = (now: number): Batch | undefined => {
275 const ready = Array.from(streams, (stream) =>
276 stream.jobs.length === 0 ? [] : [{ stream, rank: rank(stream, now) }],
277 )
278 .flat()
279 .sort((left, right) => {
280 for (let index = 0; index < left.rank.length; index++) {
281 const order = left.rank[index]! - right.rank[index]!;
282 if (order !== 0) return order;
283 }
284 return 0;
285 });
286 if (ready.length === 0) return undefined;
287 const units: Array<Unit> = [];
288 let bytes = 0;
289 let dequeued = 0;
290 let reason: Batch["reason"] = "ready";
291 for (const { stream } of ready) {
292 let take = 0;
293 while (take < stream.jobs.length && take < MAX_EVENTS - units.length) {
294 const next = stream.jobs[take]!.bytes;
295 if (units.length + take > 0 && bytes + next > MAX_BYTES) break;
296 bytes += next;
297 take++;
298 }
299 if (take < stream.jobs.length) reason = "limit";
300 if (take === 0) break;
301 const taken = stream.jobs.splice(0, take);
302 for (const job of taken) job.queued = false;
303 dequeued += take;
304 units.push(...coalesce(taken));
305 stream.servedAt = ++servings;
306 if (units.length >= MAX_EVENTS || bytes >= MAX_BYTES) break;
307 }
308 return { units, reason, dequeued };
309 };
310 // Commits a prefix of `units` in one transaction and returns how many it took.
311 const commitPrefix = (units: ReadonlyArray<Unit>, reason: Batch["reason"]) =>
312 Effect.uninterruptibleMask((restore) =>
313 Effect.gen(function* () {
314 const outcomes = new Map<Job, Exit.Exit<void, CommitBatchError>>();
315 let taken = 0;
316 const work = Effect.gen(function* () {
317 // A retried transaction runs this again from the first unit.
318 outcomes.clear();
319 taken = 0;
320 const startedAt = yield* Clock.monotonicTimeNanos;
321 for (const { job, superseded } of units) {
322 if (
323 taken > 0 &&
324 waitingWriters > 0 &&
325 (yield* Clock.monotonicTimeNanos) - startedAt >= MAX_HOLD_NANOS
326 )
327 return;
328 taken++;
329 if (!job.stream.owner.active) {
330 yield* job.cancel;
331 for (const replaced of superseded) yield* replaced.cancel;
332 continue;
333 }
334 const outcome = yield* (units.length > 1 ? isolate(job.execute) : job.execute).pipe(
335 Effect.exit,
336 );
337 outcomes.set(job, outcome);
338 if (Exit.isFailure(outcome)) {
339 job.stream.owner.active = false;
340 if (units.length === 1) return yield* Effect.failCause(outcome.cause);
341 }
342 }
343 });
344 const exit = yield* restore(
345 commit(work).pipe(Effect.provideService(CurrentWorker, yield* Effect.fiberId)),
346 ).pipe(Effect.exit);
347 const committed = taken === 0 ? units : units.slice(0, taken);
348 const jobs = committed.flatMap((unit) => [unit.job, ...unit.superseded]);
349 yield* Effect.annotateCurrentSpan({
350 "orchestration_v2.work_item_count": committed.length,
351 "orchestration_v2.coalesced_work_item_count": jobs.length - committed.length,
352 "orchestration_v2.deferred_work_item_count": units.length - committed.length,
353 "orchestration_v2.consumer_count": new Set(jobs.map((job) => job.stream)).size,
354 "orchestration_v2.raw_json_bytes": jobs.reduce((bytes, job) => bytes + job.bytes, 0),
355 "orchestration_v2.batch_flush_reason": reason,
356 "orchestration_v2.backlog_batching": true,
357 "orchestration_v2.batch_wait_ms":
358 (yield* Clock.currentTimeMillis) - Math.min(...jobs.map((job) => job.queuedAt)),
359 });
360 for (const { job, superseded } of committed) {
361 const outcome = outcomes.get(job) ?? Exit.void;
362 const final = Exit.isSuccess(exit) ? outcome : exit;
363 yield* job.settle(
364 final,
365 Exit.isSuccess(exit) ||
366 (Exit.isFailure(exit) &&
367 Exit.isFailure(outcome) &&
368 Cause.squash(exit.cause) === Cause.squash(outcome.cause)),
369 );
370 for (const replaced of superseded) yield* replaced.skip(final);
371 }
372 return committed.length;
373 }),
374 ).pipe(Effect.withSpan("orchestrationV2.ProviderCommitBatch.flush"));
375 if (enabled) {
376 yield* Effect.forever(
377 Effect.gen(function* () {
378 const batch = nextBatch(yield* Clock.currentTimeMillis);
379 if (batch === undefined) return yield* Queue.take(wake);
380 yield* backlog.release(batch.dequeued);
381 let units = batch.units;
382 while (units.length > 0) units = units.slice(yield* commitPrefix(units, batch.reason));
383 }),
384 ).pipe(Effect.ensuring(shutdown), Effect.forkScoped);
385 }
386 /**
387 * Runs `effect` after queued snapshots for `threadIds` commit; omitted, after every queued
388 * snapshot. Other threads' snapshots never delay it.
389 */
390 const barrier = Effect.fnUntraced(function* <A, E, R>(
391 effect: Effect.Effect<A, E, R>,
392 threadIds?: ReadonlyArray<string>,
393 ) {
394 if (closed) return yield* Effect.interrupt;
395 if (!enabled || (yield* CurrentWorker) === (yield* Effect.fiberId)) return yield* effect;
396 return yield* Effect.uninterruptibleMask((restore) => {
397 waitingWriters++;
398 return restore(
399 Effect.gen(function* () {
400 for (const job of Array.from(pending)) {
401 if (threadIds === undefined || threadIds.includes(job.threadId)) yield* job.settled;
402 }
403 if (closed) return yield* Effect.interrupt;
404 return yield* effect;
405 }),
406 ).pipe(
407 Effect.ensuring(
408 Effect.sync(() => {
409 waitingWriters--;
410 }),
411 ),
412 );
413 });
414 });
415 const consume = Effect.fnUntraced(function* <A extends { readonly type: string }, E, R, E2, R2>(
416 source: Stream.Stream<A, E, R>,
417 process: (event: A) => Effect.Effect<void, E2, R2>,
418 after: Effect.Effect<boolean, E2, R2>,
419 canBatch: Effect.Effect<boolean, never, R2> = Effect.succeed(true),
420 priority?: StreamPriority,
421 ) {
422 if (!enabled)
423 return yield* source.pipe(
424 Stream.mapEffect((event) => process(event).pipe(Effect.andThen(after))),
425 Stream.takeUntil(Boolean),
426 Stream.runDrain,
427 );
428 const stream: StreamState = {
429 owner: { active: true },
430 priority,
431 jobs: [],
432 unsettled: 0,
433 servedAt: 0,
434 finishing: false,
435 };
436 streams.add(stream);
437 const signals = source.pipe(
438 Stream.map((event) => ({ _tag: "Event" as const, event })),
439 Stream.catchCause((cause) => Stream.succeed({ _tag: "Failure" as const, cause })),
440 Stream.concat(Stream.succeed({ _tag: "End" as const })),
441 );
442 let resume: Effect.Effect<void> = Effect.void;
443 // Replies run in order, so EOF, failures and controls follow this stream's earlier snapshots
444 // without waiting for other streams' work.
445 yield* signals.pipe(
446 Stream.mapEffect((signal) =>
447 Effect.gen(function* () {
448 yield* resume;
449 const processed = yield* Deferred.make<void>();
450 let reply: Effect.Effect<boolean, E | E2 | CommitBatchError, R2>;
451 let check = false;
452 const snapshot =
453 signal._tag === "Event" && isSnapshot(signal.event) && (yield* canBatch)
454 ? signal.event
455 : undefined;
456 if (snapshot !== undefined) {
457 const bytes = Buffer.byteLength(yield* encodeQueue(snapshot));
458 reply = yield* enqueue(
459 process(snapshot).pipe(Effect.as(false)),
460 stream,
461 snapshotEntity(snapshot),
462 bytes,
463 false,
464 );
465 } else {
466 if (stream.unsettled > 0) stream.finishing = true;
467 if (signal._tag === "End") reply = Effect.succeed(true);
468 else if (signal._tag === "Failure") reply = Effect.failCause(signal.cause);
469 else {
470 const event = signal.event;
471 reply = Effect.sync(() => {
472 stream.finishing = false;
473 }).pipe(Effect.andThen(process(event)), Effect.as(false));
474 check = true;
475 resume = Deferred.await(processed);
476 }
477 }
478 return { reply, processed, check };
479 }),
480 ),
481 Stream.buffer({ capacity: MAX_EVENTS }),
482 Stream.mapEffect(({ reply, processed, check }) =>
483 reply.pipe(
484 Effect.flatMap((end) =>
485 end ? Effect.succeed(true) : check ? after : Effect.succeed(false),
486 ),
487 Effect.tap((stop) =>
488 Effect.sync(() => {
489 if (stop) stream.owner.active = false;
490 }),
491 ),
492 Effect.ensuring(Deferred.succeed(processed, undefined)),
493 ),
494 ),
495 Stream.takeUntil(Boolean),
496 Stream.runDrain,
497 Effect.ensuring(
498 Effect.gen(function* () {
499 stream.owner.active = false;
500 streams.delete(stream);
501 for (const job of stream.jobs.splice(0)) yield* job.cancel;
502 }),
503 ),
504 );
505 });
506 return { barrier, consume };
507 });