The best way to control Claude Code, Codex and more from any computer or phone
fix(server): compact superseded turn-item snapshots (#20990)
Yannic Charlon committed
Oct 26, 2026 at 07:40 UTC
481b9b71929b2c7c9f120712a3264fe218cf2b95
3 files changed
+500
-6
apps/server/src/orchestration-v2/FoundationPersistence.test.ts
+1
@@ -931,6 +931,7 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
931
messageEvent("message-1", "streaming"),
932
nodeEvent("node-1", "running"),
933
itemEvent("item-1", "streaming"),
934
+ itemEvent("item-middle", "still streaming"),
935
...Array.from({ length: 501 }, (_, index) =>
936
threadStateEvent(`history-${index}`, "thread.visited"),
937
),
apps/server/src/orchestration-v2/ProjectionMaintenance.ts
+11
-6
@@ -222,13 +222,12 @@ export const layer: Layer.Layer<
222
223
const supersedableThreadEventTypes = new Set(SUPERSEDABLE_THREAD_EVENT_TYPES);
224
const readStateThreadEventTypes = new Set(READ_STATE_THREAD_EVENT_TYPES);
225
- // Keep subagent.updated and turn-item.updated rows: slimmed payloads reference their prompts.
226
- // Bound each reader request and writer delete, including when the reader
227
- // worker is unavailable and discovery falls back to the writer.
225
+ // Subagent snapshots stay intact because later rows can reference an earlier prompt.
226
+ // Commanded turn items stay for command-result replay.
227
+ // Small pages bound work on the reader worker and synchronous fallback.
228
const COMPACTION_PAGE_SIZE = 50;
229
230
- /** Retain newest and oldest keyed events to preserve client insertion order; page all events to bound sparse scans.
231
- * Keep turn-item.updated because replay assigns positions on first write. */
230
+ // Keep first/latest keyed events for replay order; page every event to bound sparse scans.
231
const compactEventStore = Effect.fn("ProjectionMaintenance.compactEventStore")(function* (
232
betweenPages: Effect.Effect<void> = Effect.void,
233
) {
@@ -269,6 +268,11 @@ export const layer: Layer.Layer<
268
WHEN event.application_event_version = 2
269
AND event.event_type IN ('message.updated', 'node.updated')
270
THEN json_extract(event.payload_json, '$.id')
271
+ WHEN event.application_event_version = 2
272
+ AND event.event_type = 'turn-item.updated'
273
+ AND event.command_id IS NULL
274
+ AND json_extract(event.payload_json, '$.type') <> 'subagent'
275
+ THEN json_extract(event.payload_json, '$.id')
276
WHEN event.application_event_version = 2
277
AND event.event_type = 'thread.pull-request-link-updated'
278
THEN json_array(json_extract(event.payload_json, '$.host'),
@@ -321,7 +325,8 @@ export const layer: Layer.Layer<
325
} else if (
326
row.event_type === "thread.pull-request-link-updated" ||
327
row.event_type === "message.updated" ||
324
- row.event_type === "node.updated"
328
+ row.event_type === "node.updated" ||
329
+ (row.event_type === "turn-item.updated" && row.entity_id !== null)
330
) {
331
const key = encodeEntityKey([row.event_type, row.stream_id, row.entity_id]);
332
const candidate = oldestCandidateByEntityKey.get(key);
apps/server/src/orchestration-v2/TurnItemCompaction.test.ts
new
+488
@@ -0,0 +1,488 @@
1
+import { applyOrchestrationV2ProjectionEvent } from "../../../../packages/client-runtime/src/state/orchestrationV2Projection.ts";
2
+import { assert, it } from "@effect/vitest";
3
+import {
4
+ CommandId,
5
+ EventId,
6
+ NodeId,
7
+ PlanId,
8
+ type OrchestrationV2AppThread,
9
+ type OrchestrationV2DomainEvent,
10
+ type OrchestrationV2TurnItem,
11
+ ProjectId,
12
+ ProviderDriverKind,
13
+ ProviderInstanceId,
14
+ ThreadId,
15
+ TurnItemId,
16
+} from "@t3tools/contracts";
17
+import * as DateTime from "effect/DateTime";
18
+import * as Deferred from "effect/Deferred";
19
+import * as Effect from "effect/Effect";
20
+import * as Fiber from "effect/Fiber";
21
+import * as Layer from "effect/Layer";
22
+import * as Stream from "effect/Stream";
23
+import * as Tracer from "effect/Tracer";
24
+import * as SqlClient from "effect/sql/SqlClient";
25
+import * as Statement from "effect/sql/Statement";
26
+
27
+import * as SqlitePersistence from "../persistence/Sqlite.ts";
28
+import * as EventSink from "./EventSink.ts";
29
+import * as EventStore from "./EventStore.ts";
30
+import * as ProjectionMaintenance from "./ProjectionMaintenance.ts";
31
+import * as ProjectionStore from "./ProjectionStore.ts";
32
+
33
+const database = SqlitePersistence.layerMemory;
34
+const eventStoreProvided = EventStore.layer.pipe(Layer.provideMerge(database));
35
+const projectionStoreProvided = ProjectionStore.layer.pipe(Layer.provideMerge(database));
36
+const stores = Layer.mergeAll(database, eventStoreProvided, projectionStoreProvided);
37
+const layerTest = Layer.mergeAll(
38
+ stores,
39
+ EventSink.layer.pipe(Layer.provide(stores)),
40
+ ProjectionMaintenance.layer.pipe(Layer.provide(stores)),
41
+);
42
+
43
+const providerInstanceId = ProviderInstanceId.make("codex");
44
+const providerDriver = ProviderDriverKind.make("codex");
45
+const modelSelection = { instanceId: providerInstanceId, model: "gpt-5.4" };
46
+
47
+function makeThread(threadId: ThreadId, now: DateTime.Utc): OrchestrationV2AppThread {
48
+ return {
49
+ id: threadId,
50
+ projectId: ProjectId.make(`project:${threadId}`),
51
+ title: "Turn item compaction",
52
+ providerInstanceId,
53
+ modelSelection,
54
+ runtimeMode: "full-access",
55
+ interactionMode: "default",
56
+ branch: null,
57
+ worktreePath: null,
58
+ activeProviderThreadId: null,
59
+ lineage: { rootThreadId: threadId, parentThreadId: null, relationshipToParent: null },
60
+ forkedFrom: null,
61
+ createdBy: "user",
62
+ creationSource: "web",
63
+ createdAt: now,
64
+ updatedAt: now,
65
+ archivedAt: null,
66
+ settledOverride: null,
67
+ settledAt: null,
68
+ lastVisitedAt: null,
69
+ deletedAt: null,
70
+ };
71
+}
72
+
73
+function threadCreated(thread: OrchestrationV2AppThread, now: DateTime.Utc) {
74
+ return {
75
+ id: EventId.make("event:turn-item-compaction:thread"),
76
+ type: "thread.created",
77
+ threadId: thread.id,
78
+ providerInstanceId,
79
+ occurredAt: now,
80
+ payload: thread,
81
+ } satisfies OrchestrationV2DomainEvent;
82
+}
83
+
84
+const itemSpecs = [
85
+ { key: "command", type: "command_execution", ordinal: 10 },
86
+ { key: "reasoning", type: "reasoning", ordinal: 20 },
87
+ { key: "todo", type: "todo_list", ordinal: 30 },
88
+ { key: "commanded", type: "command_execution", ordinal: 40 },
89
+] as const;
90
+
91
+function itemEvent(
92
+ threadId: ThreadId,
93
+ now: DateTime.Utc,
94
+ spec: (typeof itemSpecs)[number],
95
+ revision: number,
96
+ eventId = `event:turn-item-compaction:${spec.key}:${revision}`,
97
+): OrchestrationV2DomainEvent {
98
+ const id = TurnItemId.make(`turn-item:compaction:${spec.key}`);
99
+ const growing = `${spec.key}:${revision}:` + "x".repeat((revision + 1) * 192);
100
+ const completed = revision === 69;
101
+ const base = {
102
+ id,
103
+ threadId,
104
+ runId: null,
105
+ nodeId: null,
106
+ providerThreadId: null,
107
+ providerTurnId: null,
108
+ nativeItemRef: null,
109
+ parentItemId: null,
110
+ ordinal: spec.ordinal,
111
+ status: completed ? ("completed" as const) : ("running" as const),
112
+ title: spec.key,
113
+ startedAt: now,
114
+ completedAt: completed ? now : null,
115
+ updatedAt: now,
116
+ };
117
+ let payload: OrchestrationV2TurnItem;
118
+ switch (spec.type) {
119
+ case "command_execution":
120
+ payload = {
121
+ ...base,
122
+ type: "command_execution",
123
+ input: `run ${spec.key}`,
124
+ output: growing,
125
+ };
126
+ break;
127
+ case "reasoning":
128
+ payload = { ...base, type: "reasoning", text: growing, streaming: !completed };
129
+ break;
130
+ case "todo_list":
131
+ payload = {
132
+ ...base,
133
+ type: "todo_list",
134
+ planId: PlanId.make("plan:turn-item-compaction"),
135
+ steps: [{ id: "step", text: growing, status: completed ? "completed" : "running" }],
136
+ explanation: growing,
137
+ };
138
+ break;
139
+ }
140
+ return {
141
+ id: EventId.make(eventId),
142
+ type: "turn-item.updated",
143
+ threadId,
144
+ occurredAt: now,
145
+ payload,
146
+ };
147
+}
148
+
149
+function subagentEvents(
150
+ threadId: ThreadId,
151
+ now: DateTime.Utc,
152
+ revision: number,
153
+): ReadonlyArray<OrchestrationV2DomainEvent> {
154
+ const id = NodeId.make("subagent:turn-item-compaction");
155
+ const prompt = revision === 0 ? "Review prompt A. ".repeat(40) : "Review prompt B. ".repeat(40);
156
+ const progress = `Review progress ${revision}`;
157
+ return [
158
+ {
159
+ id: EventId.make(`event:turn-item-compaction:subagent:${revision}`),
160
+ type: "subagent.updated",
161
+ threadId,
162
+ occurredAt: now,
163
+ payload: {
164
+ id,
165
+ threadId,
166
+ runId: null,
167
+ parentNodeId: NodeId.make("node:turn-item-compaction-parent"),
168
+ origin: "provider_native",
169
+ createdBy: "agent",
170
+ driver: providerDriver,
171
+ providerInstanceId,
172
+ providerThreadId: null,
173
+ childThreadId: null,
174
+ nativeTaskRef: {
175
+ driver: providerDriver,
176
+ nativeId: "compaction-review",
177
+ strength: "strong",
178
+ },
179
+ prompt,
180
+ progress,
181
+ title: "Review",
182
+ model: null,
183
+ status: "running",
184
+ result: null,
185
+ startedAt: now,
186
+ completedAt: null,
187
+ updatedAt: now,
188
+ },
189
+ },
190
+ {
191
+ id: EventId.make(`event:turn-item-compaction:subagent-item:${revision}`),
192
+ type: "turn-item.updated",
193
+ threadId,
194
+ occurredAt: now,
195
+ payload: {
196
+ id: TurnItemId.make("turn-item:compaction:subagent"),
197
+ threadId,
198
+ runId: null,
199
+ nodeId: null,
200
+ providerThreadId: null,
201
+ providerTurnId: null,
202
+ nativeItemRef: null,
203
+ parentItemId: null,
204
+ ordinal: 50,
205
+ status: "running",
206
+ title: "Review",
207
+ startedAt: now,
208
+ completedAt: null,
209
+ updatedAt: now,
210
+ type: "subagent",
211
+ subagentId: id,
212
+ origin: "provider_native",
213
+ driver: providerDriver,
214
+ providerInstanceId,
215
+ childThreadId: null,
216
+ prompt,
217
+ progress,
218
+ result: null,
219
+ },
220
+ },
221
+ ] satisfies ReadonlyArray<OrchestrationV2DomainEvent>;
222
+}
223
+
224
+it.effect(
225
+ "compacts ordinary snapshots while preserving replay, prompts, command lookup, and positions",
226
+ () =>
227
+ Effect.gen(function* () {
228
+ const sink = yield* EventSink.EventSinkV2;
229
+ const eventStore = yield* EventStore.EventStoreV2;
230
+ const projections = yield* ProjectionStore.ProjectionStoreV2;
231
+ const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2;
232
+ const sql = yield* SqlClient.SqlClient;
233
+ const now = yield* DateTime.now;
234
+ const threadId = ThreadId.make("thread:turn-item-compaction");
235
+ const thread = makeThread(threadId, now);
236
+ const writes = [...(yield* sink.write({ events: [threadCreated(thread, now)] }))];
237
+ const commandId = CommandId.make("command:turn-item-compaction:middle");
238
+ const snapshotsPerItem = 70;
239
+ for (let revision = 0; revision < snapshotsPerItem; revision += 1) {
240
+ const order =
241
+ revision === 0
242
+ ? [itemSpecs[0], itemSpecs[1], itemSpecs[2], itemSpecs[3]]
243
+ : [itemSpecs[1], itemSpecs[0], itemSpecs[2], itemSpecs[3]];
244
+ const events = order
245
+ .filter((spec) => spec.key !== "commanded" || revision !== 35)
246
+ .map((spec) => itemEvent(threadId, now, spec, revision));
247
+ if (revision < 3) events.push(...subagentEvents(threadId, now, revision));
248
+ const stored = yield* sink.write({ events });
249
+ writes.push(...stored);
250
+ if (revision === 35) {
251
+ const middle = itemEvent(
252
+ threadId,
253
+ now,
254
+ itemSpecs[3],
255
+ revision,
256
+ "event:turn-item-compaction:commanded:middle",
257
+ );
258
+ writes.push(...(yield* sink.write({ commandId, events: [middle] })));
259
+ }
260
+ }
261
+ const beforeProjection = yield* projections.getThreadProjection(threadId);
262
+ const highWater = yield* eventStore.latestSequence();
263
+ const bytes = yield* sql<{ readonly rows: number; readonly payload_bytes: number }>`
264
+ SELECT COUNT(*) AS rows, COALESCE(SUM(octet_length(payload_json)), 0) AS payload_bytes
265
+ FROM orchestration_events WHERE stream_id = ${threadId}
266
+ `;
267
+ const summary = yield* maintenance.compactEventStore();
268
+ const replay = yield* eventStore.read({ threadId }).pipe(Stream.runCollect);
269
+ const expectedIds = new Set([
270
+ "event:turn-item-compaction:thread",
271
+ ...itemSpecs.flatMap((spec) => [
272
+ `event:turn-item-compaction:${spec.key}:0`,
273
+ `event:turn-item-compaction:${spec.key}:${snapshotsPerItem - 1}`,
274
+ ]),
275
+ "event:turn-item-compaction:commanded:middle",
276
+ ...Array.from({ length: 3 }, (_, revision) => [
277
+ `event:turn-item-compaction:subagent:${revision}`,
278
+ `event:turn-item-compaction:subagent-item:${revision}`,
279
+ ]).flat(),
280
+ ]);
281
+ assert.deepEqual(
282
+ replay,
283
+ writes.filter((stored) => expectedIds.has(stored.event.id)),
284
+ );
285
+ assert.lengthOf(replay, 16);
286
+ assert.equal(summary.deletedEventCount, bytes[0]!.rows - replay.length);
287
+ assert.isBelow(
288
+ (yield* sql<{ readonly payload_bytes: number }>`
289
+ SELECT COALESCE(SUM(octet_length(payload_json)), 0) AS payload_bytes
290
+ FROM orchestration_events WHERE stream_id = ${threadId}
291
+ `)[0]!.payload_bytes,
292
+ bytes[0]!.payload_bytes,
293
+ );
294
+ assert.equal(yield* eventStore.latestSequence(), highWater);
295
+
296
+ const commanded = yield* sink.readByCommandId({ commandId }).pipe(Stream.runCollect);
297
+ assert.deepEqual(
298
+ commanded.map((stored) => stored.event.id),
299
+ [EventId.make("event:turn-item-compaction:commanded:middle")],
300
+ );
301
+ const storedPrompts = yield* sql<{
302
+ readonly event_id: string;
303
+ readonly sequence: number;
304
+ readonly prompt: string;
305
+ readonly prompt_sequence: number | null;
306
+ }>`
307
+ SELECT event_id, sequence,
308
+ json_extract(payload_json, '$.prompt') AS prompt,
309
+ json_extract(payload_json, '$.promptSequence') AS prompt_sequence
310
+ FROM orchestration_events
311
+ WHERE stream_id = ${threadId} AND event_type = 'subagent.updated'
312
+ ORDER BY sequence ASC
313
+ `;
314
+ assert.lengthOf(storedPrompts, 3);
315
+ assert.equal(storedPrompts[0]!.prompt_sequence, null);
316
+ assert.equal(storedPrompts[1]!.prompt_sequence, null);
317
+ assert.equal(storedPrompts[2]!.prompt, "");
318
+ assert.equal(storedPrompts[2]!.prompt_sequence, storedPrompts[1]!.sequence);
319
+
320
+ const deletedCursor = writes.find(
321
+ (stored) => stored.event.id === "event:turn-item-compaction:command:35",
322
+ )!.sequence;
323
+ assert.isFalse(replay.some((stored) => stored.sequence === deletedCursor));
324
+ const cursors = [0, deletedCursor, highWater];
325
+ for (const cursor of cursors) {
326
+ const cursorReplay = yield* eventStore
327
+ .read({ threadId, afterSequence: cursor, throughSequence: highWater })
328
+ .pipe(Stream.runCollect);
329
+ const expected = replay.filter((stored) => stored.sequence > cursor);
330
+ assert.deepEqual(
331
+ cursorReplay.map((stored) => stored.sequence),
332
+ expected.map((stored) => stored.sequence),
333
+ );
334
+
335
+ let clientProjection: typeof beforeProjection | null = {
336
+ ...beforeProjection,
337
+ subagents: [],
338
+ turnItems: [],
339
+ visibleTurnItems: [],
340
+ };
341
+ let latestLocalTurnOrdinal: number | null = null;
342
+ for (const stored of [
343
+ ...writes.filter((entry) => entry.sequence <= cursor),
344
+ ...cursorReplay,
345
+ ]) {
346
+ clientProjection = applyOrchestrationV2ProjectionEvent(clientProjection, stored.event, {
347
+ partialTimeline: true,
348
+ latestLocalTurnOrdinal,
349
+ });
350
+ if (stored.event.type === "turn-item.updated") {
351
+ latestLocalTurnOrdinal = Math.max(
352
+ latestLocalTurnOrdinal ?? 0,
353
+ stored.event.payload.ordinal,
354
+ );
355
+ }
356
+ }
357
+ assert.deepEqual(clientProjection?.turnItems, beforeProjection.turnItems);
358
+ assert.deepEqual(clientProjection?.visibleTurnItems, beforeProjection.visibleTurnItems);
359
+ }
360
+ assert.isTrue((yield* maintenance.verify).valid);
361
+ assert.isTrue((yield* maintenance.rebuild).valid);
362
+ assert.deepEqual(yield* projections.getThreadProjection(threadId), beforeProjection);
363
+ const positions = yield* sql<{
364
+ readonly turn_item_id: string;
365
+ readonly ordinal: number;
366
+ }>`
367
+ SELECT turn_item_id, ordinal
368
+ FROM orchestration_v2_turn_item_positions
369
+ WHERE thread_id = ${threadId}
370
+ ORDER BY ordinal ASC
371
+ `;
372
+ assert.deepEqual(
373
+ positions.map((row) => [row.turn_item_id, row.ordinal]),
374
+ [
375
+ ...itemSpecs.map((spec, index) => [`turn-item:compaction:${spec.key}`, index + 1]),
376
+ ["turn-item:compaction:subagent", 5],
377
+ ],
378
+ );
379
+ }).pipe(Effect.provide(layerTest)),
380
+);
381
+
382
+it.effect("keeps post-bound appends visible during compaction", () =>
383
+ Effect.scoped(
384
+ Effect.gen(function* () {
385
+ const sink = yield* EventSink.EventSinkV2;
386
+ const eventStore = yield* EventStore.EventStoreV2;
387
+ const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2;
388
+ const now = yield* DateTime.now;
389
+ const threadId = ThreadId.make("thread:turn-item-compaction-race");
390
+ const thread = makeThread(threadId, now);
391
+ const initial = yield* sink.write({
392
+ events: [
393
+ threadCreated(thread, now),
394
+ ...[0, 1, 2].map((revision) => itemEvent(threadId, now, itemSpecs[0], revision)),
395
+ ],
396
+ });
397
+ const beforeSequence = initial.at(-1)!.sequence;
398
+
399
+ const streamReady = yield* Deferred.make<void>();
400
+ const streamTracer = Tracer.make({
401
+ span(options) {
402
+ const span = new Tracer.NativeSpan(options);
403
+ const end = span.end.bind(span);
404
+ span.end = (endTime, exit) => {
405
+ end(endTime, exit);
406
+ const query = span.attributes.get("db.query.text");
407
+ if (typeof query === "string" && query.includes("MAX(sequence)")) {
408
+ Deferred.doneUnsafe(streamReady, Effect.void);
409
+ }
410
+ };
411
+ return span;
412
+ },
413
+ });
414
+ const liveEvents = yield* sink
415
+ .stream({ threadId, afterSequence: beforeSequence })
416
+ .pipe(
417
+ Stream.take(3),
418
+ Stream.runCollect,
419
+ Effect.withTracer(streamTracer),
420
+ Effect.forkScoped,
421
+ );
422
+ yield* Deferred.await(streamReady);
423
+
424
+ const scanStarted = yield* Deferred.make<void>();
425
+ const releaseScan = yield* Deferred.make<void>();
426
+ let blocked = false;
427
+ const blockFirstScanPage: Statement.Transformer = (statement) => {
428
+ const [query] = statement.compile();
429
+ if (
430
+ !blocked &&
431
+ query.includes("FROM orchestration_events AS event") &&
432
+ query.includes("ORDER BY event.sequence DESC")
433
+ ) {
434
+ blocked = true;
435
+ return Deferred.succeed(scanStarted, undefined).pipe(
436
+ Effect.andThen(Deferred.await(releaseScan)),
437
+ Effect.as(statement),
438
+ );
439
+ }
440
+ return Effect.succeed(statement);
441
+ };
442
+ const compaction = yield* maintenance
443
+ .compactEventStore()
444
+ .pipe(
445
+ Effect.provideService(Statement.CurrentTransformer, blockFirstScanPage),
446
+ Effect.forkScoped,
447
+ );
448
+ yield* Deferred.await(scanStarted);
449
+ const snapshots = [3, 4, 5].map((revision) =>
450
+ itemEvent(
451
+ threadId,
452
+ now,
453
+ itemSpecs[0],
454
+ revision,
455
+ `event:turn-item-compaction-race:${revision}`,
456
+ ),
457
+ );
458
+ const appended = yield* sink.write({ events: snapshots });
459
+ yield* Deferred.succeed(releaseScan, undefined);
460
+ assert.equal((yield* Fiber.join(compaction)).deletedEventCount, 1);
461
+ const streamed = yield* Fiber.join(liveEvents);
462
+ assert.deepEqual(
463
+ streamed.map((stored) => stored.sequence),
464
+ appended.map((stored) => stored.sequence),
465
+ );
466
+ assert.deepEqual(
467
+ streamed.map((stored) => stored.event.id),
468
+ snapshots.map((event) => event.id),
469
+ );
470
+ assert.equal(yield* eventStore.latestSequence(), appended.at(-1)!.sequence);
471
+ const firstPassRows = yield* eventStore
472
+ .read({ threadId, eventType: "turn-item.updated" })
473
+ .pipe(Stream.runCollect);
474
+ assert.deepEqual(
475
+ firstPassRows.map((stored) => stored.sequence),
476
+ [initial[1]!.sequence, beforeSequence, ...appended.map((stored) => stored.sequence)],
477
+ );
478
+ yield* maintenance.compactEventStore();
479
+ const secondPassRows = yield* eventStore
480
+ .read({ threadId, eventType: "turn-item.updated" })
481
+ .pipe(Stream.runCollect);
482
+ assert.deepEqual(
483
+ secondPassRows.map((stored) => stored.sequence),
484
+ [initial[1]!.sequence, appended.at(-1)!.sequence],
485
+ );
486
+ }),
487
+ ).pipe(Effect.provide(layerTest)),
488
+);