@devries-cameron20 / t3code-1 / commits / 481b9b7192
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): compact superseded turn-item snapshots (#20990)

Yannic Charlon committed Oct 26, 2026 at 07:40 UTC 481b9b71929b2c7c9f120712a3264fe218cf2b95
3 files changed +500 -6
apps/server/src/orchestration-v2/FoundationPersistence.test.ts
+1
@@ -931,6 +931,7 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
931 messageEvent("message-1", "streaming"),
932 nodeEvent("node-1", "running"),
933 itemEvent("item-1", "streaming"),
934 + itemEvent("item-middle", "still streaming"),
935 ...Array.from({ length: 501 }, (_, index) =>
936 threadStateEvent(`history-${index}`, "thread.visited"),
937 ),
apps/server/src/orchestration-v2/ProjectionMaintenance.ts
+11 -6
@@ -222,13 +222,12 @@ export const layer: Layer.Layer<
222
223 const supersedableThreadEventTypes = new Set(SUPERSEDABLE_THREAD_EVENT_TYPES);
224 const readStateThreadEventTypes = new Set(READ_STATE_THREAD_EVENT_TYPES);
225 - // Keep subagent.updated and turn-item.updated rows: slimmed payloads reference their prompts.
226 - // Bound each reader request and writer delete, including when the reader
227 - // worker is unavailable and discovery falls back to the writer.
225 + // Subagent snapshots stay intact because later rows can reference an earlier prompt.
226 + // Commanded turn items stay for command-result replay.
227 + // Small pages bound work on the reader worker and synchronous fallback.
228 const COMPACTION_PAGE_SIZE = 50;
229
230 - /** Retain newest and oldest keyed events to preserve client insertion order; page all events to bound sparse scans.
231 - * Keep turn-item.updated because replay assigns positions on first write. */
230 + // Keep first/latest keyed events for replay order; page every event to bound sparse scans.
231 const compactEventStore = Effect.fn("ProjectionMaintenance.compactEventStore")(function* (
232 betweenPages: Effect.Effect<void> = Effect.void,
233 ) {
@@ -269,6 +268,11 @@ export const layer: Layer.Layer<
268 WHEN event.application_event_version = 2
269 AND event.event_type IN ('message.updated', 'node.updated')
270 THEN json_extract(event.payload_json, '$.id')
271 + WHEN event.application_event_version = 2
272 + AND event.event_type = 'turn-item.updated'
273 + AND event.command_id IS NULL
274 + AND json_extract(event.payload_json, '$.type') <> 'subagent'
275 + THEN json_extract(event.payload_json, '$.id')
276 WHEN event.application_event_version = 2
277 AND event.event_type = 'thread.pull-request-link-updated'
278 THEN json_array(json_extract(event.payload_json, '$.host'),
@@ -321,7 +325,8 @@ export const layer: Layer.Layer<
325 } else if (
326 row.event_type === "thread.pull-request-link-updated" ||
327 row.event_type === "message.updated" ||
324 - row.event_type === "node.updated"
328 + row.event_type === "node.updated" ||
329 + (row.event_type === "turn-item.updated" && row.entity_id !== null)
330 ) {
331 const key = encodeEntityKey([row.event_type, row.stream_id, row.entity_id]);
332 const candidate = oldestCandidateByEntityKey.get(key);
apps/server/src/orchestration-v2/TurnItemCompaction.test.ts new
+488
@@ -0,0 +1,488 @@
1 +import { applyOrchestrationV2ProjectionEvent } from "../../../../packages/client-runtime/src/state/orchestrationV2Projection.ts";
2 +import { assert, it } from "@effect/vitest";
3 +import {
4 + CommandId,
5 + EventId,
6 + NodeId,
7 + PlanId,
8 + type OrchestrationV2AppThread,
9 + type OrchestrationV2DomainEvent,
10 + type OrchestrationV2TurnItem,
11 + ProjectId,
12 + ProviderDriverKind,
13 + ProviderInstanceId,
14 + ThreadId,
15 + TurnItemId,
16 +} from "@t3tools/contracts";
17 +import * as DateTime from "effect/DateTime";
18 +import * as Deferred from "effect/Deferred";
19 +import * as Effect from "effect/Effect";
20 +import * as Fiber from "effect/Fiber";
21 +import * as Layer from "effect/Layer";
22 +import * as Stream from "effect/Stream";
23 +import * as Tracer from "effect/Tracer";
24 +import * as SqlClient from "effect/sql/SqlClient";
25 +import * as Statement from "effect/sql/Statement";
26 +
27 +import * as SqlitePersistence from "../persistence/Sqlite.ts";
28 +import * as EventSink from "./EventSink.ts";
29 +import * as EventStore from "./EventStore.ts";
30 +import * as ProjectionMaintenance from "./ProjectionMaintenance.ts";
31 +import * as ProjectionStore from "./ProjectionStore.ts";
32 +
33 +const database = SqlitePersistence.layerMemory;
34 +const eventStoreProvided = EventStore.layer.pipe(Layer.provideMerge(database));
35 +const projectionStoreProvided = ProjectionStore.layer.pipe(Layer.provideMerge(database));
36 +const stores = Layer.mergeAll(database, eventStoreProvided, projectionStoreProvided);
37 +const layerTest = Layer.mergeAll(
38 + stores,
39 + EventSink.layer.pipe(Layer.provide(stores)),
40 + ProjectionMaintenance.layer.pipe(Layer.provide(stores)),
41 +);
42 +
43 +const providerInstanceId = ProviderInstanceId.make("codex");
44 +const providerDriver = ProviderDriverKind.make("codex");
45 +const modelSelection = { instanceId: providerInstanceId, model: "gpt-5.4" };
46 +
47 +function makeThread(threadId: ThreadId, now: DateTime.Utc): OrchestrationV2AppThread {
48 + return {
49 + id: threadId,
50 + projectId: ProjectId.make(`project:${threadId}`),
51 + title: "Turn item compaction",
52 + providerInstanceId,
53 + modelSelection,
54 + runtimeMode: "full-access",
55 + interactionMode: "default",
56 + branch: null,
57 + worktreePath: null,
58 + activeProviderThreadId: null,
59 + lineage: { rootThreadId: threadId, parentThreadId: null, relationshipToParent: null },
60 + forkedFrom: null,
61 + createdBy: "user",
62 + creationSource: "web",
63 + createdAt: now,
64 + updatedAt: now,
65 + archivedAt: null,
66 + settledOverride: null,
67 + settledAt: null,
68 + lastVisitedAt: null,
69 + deletedAt: null,
70 + };
71 +}
72 +
73 +function threadCreated(thread: OrchestrationV2AppThread, now: DateTime.Utc) {
74 + return {
75 + id: EventId.make("event:turn-item-compaction:thread"),
76 + type: "thread.created",
77 + threadId: thread.id,
78 + providerInstanceId,
79 + occurredAt: now,
80 + payload: thread,
81 + } satisfies OrchestrationV2DomainEvent;
82 +}
83 +
84 +const itemSpecs = [
85 + { key: "command", type: "command_execution", ordinal: 10 },
86 + { key: "reasoning", type: "reasoning", ordinal: 20 },
87 + { key: "todo", type: "todo_list", ordinal: 30 },
88 + { key: "commanded", type: "command_execution", ordinal: 40 },
89 +] as const;
90 +
91 +function itemEvent(
92 + threadId: ThreadId,
93 + now: DateTime.Utc,
94 + spec: (typeof itemSpecs)[number],
95 + revision: number,
96 + eventId = `event:turn-item-compaction:${spec.key}:${revision}`,
97 +): OrchestrationV2DomainEvent {
98 + const id = TurnItemId.make(`turn-item:compaction:${spec.key}`);
99 + const growing = `${spec.key}:${revision}:` + "x".repeat((revision + 1) * 192);
100 + const completed = revision === 69;
101 + const base = {
102 + id,
103 + threadId,
104 + runId: null,
105 + nodeId: null,
106 + providerThreadId: null,
107 + providerTurnId: null,
108 + nativeItemRef: null,
109 + parentItemId: null,
110 + ordinal: spec.ordinal,
111 + status: completed ? ("completed" as const) : ("running" as const),
112 + title: spec.key,
113 + startedAt: now,
114 + completedAt: completed ? now : null,
115 + updatedAt: now,
116 + };
117 + let payload: OrchestrationV2TurnItem;
118 + switch (spec.type) {
119 + case "command_execution":
120 + payload = {
121 + ...base,
122 + type: "command_execution",
123 + input: `run ${spec.key}`,
124 + output: growing,
125 + };
126 + break;
127 + case "reasoning":
128 + payload = { ...base, type: "reasoning", text: growing, streaming: !completed };
129 + break;
130 + case "todo_list":
131 + payload = {
132 + ...base,
133 + type: "todo_list",
134 + planId: PlanId.make("plan:turn-item-compaction"),
135 + steps: [{ id: "step", text: growing, status: completed ? "completed" : "running" }],
136 + explanation: growing,
137 + };
138 + break;
139 + }
140 + return {
141 + id: EventId.make(eventId),
142 + type: "turn-item.updated",
143 + threadId,
144 + occurredAt: now,
145 + payload,
146 + };
147 +}
148 +
149 +function subagentEvents(
150 + threadId: ThreadId,
151 + now: DateTime.Utc,
152 + revision: number,
153 +): ReadonlyArray<OrchestrationV2DomainEvent> {
154 + const id = NodeId.make("subagent:turn-item-compaction");
155 + const prompt = revision === 0 ? "Review prompt A. ".repeat(40) : "Review prompt B. ".repeat(40);
156 + const progress = `Review progress ${revision}`;
157 + return [
158 + {
159 + id: EventId.make(`event:turn-item-compaction:subagent:${revision}`),
160 + type: "subagent.updated",
161 + threadId,
162 + occurredAt: now,
163 + payload: {
164 + id,
165 + threadId,
166 + runId: null,
167 + parentNodeId: NodeId.make("node:turn-item-compaction-parent"),
168 + origin: "provider_native",
169 + createdBy: "agent",
170 + driver: providerDriver,
171 + providerInstanceId,
172 + providerThreadId: null,
173 + childThreadId: null,
174 + nativeTaskRef: {
175 + driver: providerDriver,
176 + nativeId: "compaction-review",
177 + strength: "strong",
178 + },
179 + prompt,
180 + progress,
181 + title: "Review",
182 + model: null,
183 + status: "running",
184 + result: null,
185 + startedAt: now,
186 + completedAt: null,
187 + updatedAt: now,
188 + },
189 + },
190 + {
191 + id: EventId.make(`event:turn-item-compaction:subagent-item:${revision}`),
192 + type: "turn-item.updated",
193 + threadId,
194 + occurredAt: now,
195 + payload: {
196 + id: TurnItemId.make("turn-item:compaction:subagent"),
197 + threadId,
198 + runId: null,
199 + nodeId: null,
200 + providerThreadId: null,
201 + providerTurnId: null,
202 + nativeItemRef: null,
203 + parentItemId: null,
204 + ordinal: 50,
205 + status: "running",
206 + title: "Review",
207 + startedAt: now,
208 + completedAt: null,
209 + updatedAt: now,
210 + type: "subagent",
211 + subagentId: id,
212 + origin: "provider_native",
213 + driver: providerDriver,
214 + providerInstanceId,
215 + childThreadId: null,
216 + prompt,
217 + progress,
218 + result: null,
219 + },
220 + },
221 + ] satisfies ReadonlyArray<OrchestrationV2DomainEvent>;
222 +}
223 +
224 +it.effect(
225 + "compacts ordinary snapshots while preserving replay, prompts, command lookup, and positions",
226 + () =>
227 + Effect.gen(function* () {
228 + const sink = yield* EventSink.EventSinkV2;
229 + const eventStore = yield* EventStore.EventStoreV2;
230 + const projections = yield* ProjectionStore.ProjectionStoreV2;
231 + const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2;
232 + const sql = yield* SqlClient.SqlClient;
233 + const now = yield* DateTime.now;
234 + const threadId = ThreadId.make("thread:turn-item-compaction");
235 + const thread = makeThread(threadId, now);
236 + const writes = [...(yield* sink.write({ events: [threadCreated(thread, now)] }))];
237 + const commandId = CommandId.make("command:turn-item-compaction:middle");
238 + const snapshotsPerItem = 70;
239 + for (let revision = 0; revision < snapshotsPerItem; revision += 1) {
240 + const order =
241 + revision === 0
242 + ? [itemSpecs[0], itemSpecs[1], itemSpecs[2], itemSpecs[3]]
243 + : [itemSpecs[1], itemSpecs[0], itemSpecs[2], itemSpecs[3]];
244 + const events = order
245 + .filter((spec) => spec.key !== "commanded" || revision !== 35)
246 + .map((spec) => itemEvent(threadId, now, spec, revision));
247 + if (revision < 3) events.push(...subagentEvents(threadId, now, revision));
248 + const stored = yield* sink.write({ events });
249 + writes.push(...stored);
250 + if (revision === 35) {
251 + const middle = itemEvent(
252 + threadId,
253 + now,
254 + itemSpecs[3],
255 + revision,
256 + "event:turn-item-compaction:commanded:middle",
257 + );
258 + writes.push(...(yield* sink.write({ commandId, events: [middle] })));
259 + }
260 + }
261 + const beforeProjection = yield* projections.getThreadProjection(threadId);
262 + const highWater = yield* eventStore.latestSequence();
263 + const bytes = yield* sql<{ readonly rows: number; readonly payload_bytes: number }>`
264 + SELECT COUNT(*) AS rows, COALESCE(SUM(octet_length(payload_json)), 0) AS payload_bytes
265 + FROM orchestration_events WHERE stream_id = ${threadId}
266 + `;
267 + const summary = yield* maintenance.compactEventStore();
268 + const replay = yield* eventStore.read({ threadId }).pipe(Stream.runCollect);
269 + const expectedIds = new Set([
270 + "event:turn-item-compaction:thread",
271 + ...itemSpecs.flatMap((spec) => [
272 + `event:turn-item-compaction:${spec.key}:0`,
273 + `event:turn-item-compaction:${spec.key}:${snapshotsPerItem - 1}`,
274 + ]),
275 + "event:turn-item-compaction:commanded:middle",
276 + ...Array.from({ length: 3 }, (_, revision) => [
277 + `event:turn-item-compaction:subagent:${revision}`,
278 + `event:turn-item-compaction:subagent-item:${revision}`,
279 + ]).flat(),
280 + ]);
281 + assert.deepEqual(
282 + replay,
283 + writes.filter((stored) => expectedIds.has(stored.event.id)),
284 + );
285 + assert.lengthOf(replay, 16);
286 + assert.equal(summary.deletedEventCount, bytes[0]!.rows - replay.length);
287 + assert.isBelow(
288 + (yield* sql<{ readonly payload_bytes: number }>`
289 + SELECT COALESCE(SUM(octet_length(payload_json)), 0) AS payload_bytes
290 + FROM orchestration_events WHERE stream_id = ${threadId}
291 + `)[0]!.payload_bytes,
292 + bytes[0]!.payload_bytes,
293 + );
294 + assert.equal(yield* eventStore.latestSequence(), highWater);
295 +
296 + const commanded = yield* sink.readByCommandId({ commandId }).pipe(Stream.runCollect);
297 + assert.deepEqual(
298 + commanded.map((stored) => stored.event.id),
299 + [EventId.make("event:turn-item-compaction:commanded:middle")],
300 + );
301 + const storedPrompts = yield* sql<{
302 + readonly event_id: string;
303 + readonly sequence: number;
304 + readonly prompt: string;
305 + readonly prompt_sequence: number | null;
306 + }>`
307 + SELECT event_id, sequence,
308 + json_extract(payload_json, '$.prompt') AS prompt,
309 + json_extract(payload_json, '$.promptSequence') AS prompt_sequence
310 + FROM orchestration_events
311 + WHERE stream_id = ${threadId} AND event_type = 'subagent.updated'
312 + ORDER BY sequence ASC
313 + `;
314 + assert.lengthOf(storedPrompts, 3);
315 + assert.equal(storedPrompts[0]!.prompt_sequence, null);
316 + assert.equal(storedPrompts[1]!.prompt_sequence, null);
317 + assert.equal(storedPrompts[2]!.prompt, "");
318 + assert.equal(storedPrompts[2]!.prompt_sequence, storedPrompts[1]!.sequence);
319 +
320 + const deletedCursor = writes.find(
321 + (stored) => stored.event.id === "event:turn-item-compaction:command:35",
322 + )!.sequence;
323 + assert.isFalse(replay.some((stored) => stored.sequence === deletedCursor));
324 + const cursors = [0, deletedCursor, highWater];
325 + for (const cursor of cursors) {
326 + const cursorReplay = yield* eventStore
327 + .read({ threadId, afterSequence: cursor, throughSequence: highWater })
328 + .pipe(Stream.runCollect);
329 + const expected = replay.filter((stored) => stored.sequence > cursor);
330 + assert.deepEqual(
331 + cursorReplay.map((stored) => stored.sequence),
332 + expected.map((stored) => stored.sequence),
333 + );
334 +
335 + let clientProjection: typeof beforeProjection | null = {
336 + ...beforeProjection,
337 + subagents: [],
338 + turnItems: [],
339 + visibleTurnItems: [],
340 + };
341 + let latestLocalTurnOrdinal: number | null = null;
342 + for (const stored of [
343 + ...writes.filter((entry) => entry.sequence <= cursor),
344 + ...cursorReplay,
345 + ]) {
346 + clientProjection = applyOrchestrationV2ProjectionEvent(clientProjection, stored.event, {
347 + partialTimeline: true,
348 + latestLocalTurnOrdinal,
349 + });
350 + if (stored.event.type === "turn-item.updated") {
351 + latestLocalTurnOrdinal = Math.max(
352 + latestLocalTurnOrdinal ?? 0,
353 + stored.event.payload.ordinal,
354 + );
355 + }
356 + }
357 + assert.deepEqual(clientProjection?.turnItems, beforeProjection.turnItems);
358 + assert.deepEqual(clientProjection?.visibleTurnItems, beforeProjection.visibleTurnItems);
359 + }
360 + assert.isTrue((yield* maintenance.verify).valid);
361 + assert.isTrue((yield* maintenance.rebuild).valid);
362 + assert.deepEqual(yield* projections.getThreadProjection(threadId), beforeProjection);
363 + const positions = yield* sql<{
364 + readonly turn_item_id: string;
365 + readonly ordinal: number;
366 + }>`
367 + SELECT turn_item_id, ordinal
368 + FROM orchestration_v2_turn_item_positions
369 + WHERE thread_id = ${threadId}
370 + ORDER BY ordinal ASC
371 + `;
372 + assert.deepEqual(
373 + positions.map((row) => [row.turn_item_id, row.ordinal]),
374 + [
375 + ...itemSpecs.map((spec, index) => [`turn-item:compaction:${spec.key}`, index + 1]),
376 + ["turn-item:compaction:subagent", 5],
377 + ],
378 + );
379 + }).pipe(Effect.provide(layerTest)),
380 +);
381 +
382 +it.effect("keeps post-bound appends visible during compaction", () =>
383 + Effect.scoped(
384 + Effect.gen(function* () {
385 + const sink = yield* EventSink.EventSinkV2;
386 + const eventStore = yield* EventStore.EventStoreV2;
387 + const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2;
388 + const now = yield* DateTime.now;
389 + const threadId = ThreadId.make("thread:turn-item-compaction-race");
390 + const thread = makeThread(threadId, now);
391 + const initial = yield* sink.write({
392 + events: [
393 + threadCreated(thread, now),
394 + ...[0, 1, 2].map((revision) => itemEvent(threadId, now, itemSpecs[0], revision)),
395 + ],
396 + });
397 + const beforeSequence = initial.at(-1)!.sequence;
398 +
399 + const streamReady = yield* Deferred.make<void>();
400 + const streamTracer = Tracer.make({
401 + span(options) {
402 + const span = new Tracer.NativeSpan(options);
403 + const end = span.end.bind(span);
404 + span.end = (endTime, exit) => {
405 + end(endTime, exit);
406 + const query = span.attributes.get("db.query.text");
407 + if (typeof query === "string" && query.includes("MAX(sequence)")) {
408 + Deferred.doneUnsafe(streamReady, Effect.void);
409 + }
410 + };
411 + return span;
412 + },
413 + });
414 + const liveEvents = yield* sink
415 + .stream({ threadId, afterSequence: beforeSequence })
416 + .pipe(
417 + Stream.take(3),
418 + Stream.runCollect,
419 + Effect.withTracer(streamTracer),
420 + Effect.forkScoped,
421 + );
422 + yield* Deferred.await(streamReady);
423 +
424 + const scanStarted = yield* Deferred.make<void>();
425 + const releaseScan = yield* Deferred.make<void>();
426 + let blocked = false;
427 + const blockFirstScanPage: Statement.Transformer = (statement) => {
428 + const [query] = statement.compile();
429 + if (
430 + !blocked &&
431 + query.includes("FROM orchestration_events AS event") &&
432 + query.includes("ORDER BY event.sequence DESC")
433 + ) {
434 + blocked = true;
435 + return Deferred.succeed(scanStarted, undefined).pipe(
436 + Effect.andThen(Deferred.await(releaseScan)),
437 + Effect.as(statement),
438 + );
439 + }
440 + return Effect.succeed(statement);
441 + };
442 + const compaction = yield* maintenance
443 + .compactEventStore()
444 + .pipe(
445 + Effect.provideService(Statement.CurrentTransformer, blockFirstScanPage),
446 + Effect.forkScoped,
447 + );
448 + yield* Deferred.await(scanStarted);
449 + const snapshots = [3, 4, 5].map((revision) =>
450 + itemEvent(
451 + threadId,
452 + now,
453 + itemSpecs[0],
454 + revision,
455 + `event:turn-item-compaction-race:${revision}`,
456 + ),
457 + );
458 + const appended = yield* sink.write({ events: snapshots });
459 + yield* Deferred.succeed(releaseScan, undefined);
460 + assert.equal((yield* Fiber.join(compaction)).deletedEventCount, 1);
461 + const streamed = yield* Fiber.join(liveEvents);
462 + assert.deepEqual(
463 + streamed.map((stored) => stored.sequence),
464 + appended.map((stored) => stored.sequence),
465 + );
466 + assert.deepEqual(
467 + streamed.map((stored) => stored.event.id),
468 + snapshots.map((event) => event.id),
469 + );
470 + assert.equal(yield* eventStore.latestSequence(), appended.at(-1)!.sequence);
471 + const firstPassRows = yield* eventStore
472 + .read({ threadId, eventType: "turn-item.updated" })
473 + .pipe(Stream.runCollect);
474 + assert.deepEqual(
475 + firstPassRows.map((stored) => stored.sequence),
476 + [initial[1]!.sequence, beforeSequence, ...appended.map((stored) => stored.sequence)],
477 + );
478 + yield* maintenance.compactEventStore();
479 + const secondPassRows = yield* eventStore
480 + .read({ threadId, eventType: "turn-item.updated" })
481 + .pipe(Stream.runCollect);
482 + assert.deepEqual(
483 + secondPassRows.map((stored) => stored.sequence),
484 + [initial[1]!.sequence, appended.at(-1)!.sequence],
485 + );
486 + }),
487 + ).pipe(Effect.provide(layerTest)),
488 +);