The best way to control Claude Code, Codex and more from any computer or phone
perf(server): sending to a long compacted thread waits about 3x less on its history read (#35180)
Replaces #34641. Refs #29468. Co-authored-by: Braulio Oliveira <brauliobo@gmail.com> Co-authored-by: Theo Browne <me@theo.li> Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com>
Theo Browne committed
Nov 12, 2026 at 04:55 UTC
591188471a6d8df7dad63083670afb6c2fb4d100
4 files changed
+665
-14
apps/server/src/orchestration-v2/Orchestrator.ts
+56
-6
@@ -1528,7 +1528,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1528
});
1529
1530
// Runs in every command transaction; avoid reading thousands of finished leaf nodes.
1531
- const readCommandProjection = (threadId: ThreadId) =>
1531
+ const readCommandProjection = (
1532
+ threadId: ThreadId,
1533
+ recent?: ProjectionRecordFilter["sinceLastCompaction"],
1534
+ ) =>
1535
projectionStore
1536
.getThreadRecords(
1537
threadId,
@@ -1557,6 +1560,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1560
omitSettledSubagentRecords: true,
1561
// Commands act on pending forks and merge-backs only; consumed ones stay unread.
1562
contextTransferStatuses: ["pending"],
1563
+ ...(recent === undefined ? {} : { sinceLastCompaction: recent }),
1564
},
1565
)
1566
.pipe(
@@ -1645,9 +1649,10 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1649
Effect.fnUntraced(function* (
1650
threadId: ThreadId,
1651
events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
1652
+ recent?: ProjectionRecordFilter["sinceLastCompaction"],
1653
) {
1654
const pending = (yield* Ref.get(events)).filter((event) => event.threadId === threadId);
1650
- const stored = yield* Effect.option(readCommandProjection(threadId));
1655
+ const stored = yield* Effect.option(readCommandProjection(threadId, recent));
1656
let projection: OrchestrationV2ThreadProjection;
1657
if (Option.isSome(stored)) {
1658
projection = stored.value;
@@ -6307,7 +6312,28 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
6312
events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
6313
effects: Ref.Ref<Array<PendingOrchestrationEffectV2>>,
6314
) {
6310
- let projection = yield* getProjectionWithPendingEvents(command.threadId, events);
6315
+ // A compacted thread reads every run and turn item, but attempts, nodes, messages and provider
6316
+ // turns only for its recent runs and the runs this command names. An idle compaction reads
6317
+ // in full: its busy check below looks for a delegated result's message under any old run.
6318
+ let projection = yield* getProjectionWithPendingEvents(
6319
+ command.threadId,
6320
+ events,
6321
+ command.idleCompaction === undefined
6322
+ ? {
6323
+ keepRunIds: [
6324
+ command.manualContinuationOfRunId,
6325
+ command.restartContinuationOfRunId,
6326
+ command.usageLimitContinuationOfRunId,
6327
+ command.steerFollowUpOfRunId,
6328
+ ...(command.dispatchMode.type === "steer_active" ||
6329
+ command.dispatchMode.type === "restart_active"
6330
+ ? [command.dispatchMode.targetRunId]
6331
+ : []),
6332
+ ].flatMap((runId) => (runId === undefined ? [] : [runId])),
6333
+ keepMessageId: command.messageId,
6334
+ }
6335
+ : undefined,
6336
+ );
6337
if (command.steerFollowUpOfRunId !== undefined) {
6338
const source = projection.runs.find((run) => run.id === command.steerFollowUpOfRunId);
6339
if (
@@ -6344,6 +6370,18 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
6370
});
6371
return;
6372
}
6373
+ const activeProviderInstanceId = projection.providerThreads.find(
6374
+ (candidate) => candidate.id === projection.thread.activeProviderThreadId,
6375
+ )?.providerInstanceId;
6376
+ // Another provider is handed what ran since it last saw the thread, which can be older than
6377
+ // the compaction, so a provider switch reads the whole history.
6378
+ if (
6379
+ activeProviderInstanceId !== undefined &&
6380
+ activeProviderInstanceId !==
6381
+ (command.modelSelection ?? projection.thread.modelSelection).instanceId
6382
+ ) {
6383
+ projection = yield* getProjectionWithPendingEvents(command.threadId, events);
6384
+ }
6385
if (command.manualContinuationOfRunId !== undefined) {
6386
const source = projection.runs.find((run) => run.id === command.manualContinuationOfRunId);
6387
const limited = latestRootProviderFailure(source ?? null, projection.turnItems);
@@ -6547,15 +6585,27 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
6585
projection = applyToProjection(projection, event);
6586
}
6587
const userMessages = projection.messages.filter((message) => message.role === "user");
6550
- const onlyMaintenanceHistory =
6551
- userMessages.length > 0 && userMessages.every(isNativeMaintenanceCommand);
6588
+ // The recent runs of a compacted thread can hold only a `/compact` while an older prompt
6589
+ // exists. Every prompt's turn item is loaded; without one, every user message is checked.
6590
+ const onlyMaintenanceHistory = [
6591
+ ...userMessages,
6592
+ ...projection.turnItems.flatMap((item) => (item.type === "user_message" ? [item] : [])),
6593
+ ].every(isNativeMaintenanceCommand)
6594
+ ? projectionStore
6595
+ .getThreadRecords(command.threadId, ["messages"], { messageRoles: ["user"] })
6596
+ .pipe(
6597
+ Effect.map(({ messages }) => [...messages, ...userMessages]),
6598
+ Effect.map((all) => all.length > 0 && all.every(isNativeMaintenanceCommand)),
6599
+ mapDispatchError(command),
6600
+ )
6601
+ : Effect.succeed(false);
6602
if (
6603
!isNativeMaintenanceCommand(command) &&
6604
((command.titleSeed !== undefined &&
6605
(yield* projectionStore
6606
.getMessageCount(command.threadId)
6607
.pipe(mapDispatchError(command))) === 0) ||
6558
- (command.createdBy === "user" && onlyMaintenanceHistory))
6608
+ (command.createdBy === "user" && (yield* onlyMaintenanceHistory)))
6609
) {
6610
const now = yield* DateTime.now;
6611
const thread: OrchestrationV2AppThread = {
apps/server/src/orchestration-v2/ProjectionStore.test.ts
+435
@@ -795,6 +795,441 @@ it.effect("idle compaction filters keep the latest run, live work and the newest
795
}).pipe(Effect.provide(layerTest)),
796
);
797
798
+// One thread holds every way a run can still matter after a compaction. A run is requested at
799
+// its ordinal in minutes and ends 30 seconds later, unless the table says otherwise.
800
+it.effect("a compacted thread reads every run, and run records only for its recent runs", () =>
801
+ Effect.gen(function* () {
802
+ const store = yield* ProjectionStore.ProjectionStoreV2;
803
+ const threadId = yield* addRolledBackRecoveryCandidate("compacted");
804
+ const created = yield* DateTime.now;
805
+ const at = (minutes: number) => DateTime.add(created, { minutes });
806
+ const runId = (ordinal: number) => RunId.make(`run:compacted:${ordinal}`);
807
+ const rootNodeId = (ordinal: number) => NodeId.make(`node:compacted:${ordinal}`);
808
+ const attemptId = (ordinal: number) => RunAttemptId.make(`attempt:compacted:${ordinal}`);
809
+ const providerThreadId = ProviderThreadId.make("provider-thread:compacted");
810
+ const messageId = (ordinal: number) => MessageId.make(`message:compacted:${ordinal}`);
811
+ const item = (
812
+ ordinal: number,
813
+ itemOrdinal: number,
814
+ status: "completed" | "failed" | "running",
815
+ ) => ({
816
+ threadId,
817
+ runId: runId(ordinal),
818
+ nodeId: rootNodeId(ordinal),
819
+ providerThreadId: null,
820
+ providerTurnId: null,
821
+ nativeItemRef: null,
822
+ parentItemId: null,
823
+ ordinal: itemOrdinal,
824
+ status,
825
+ title: null,
826
+ startedAt: at(ordinal),
827
+ completedAt: status === "running" ? null : at(ordinal),
828
+ updatedAt: at(ordinal),
829
+ });
830
+ const compaction = (ordinal: number, status: "completed" | "failed" | "running") =>
831
+ store.apply({
832
+ id: EventId.make(`event:compacted:compaction-${ordinal}`),
833
+ type: "turn-item.updated",
834
+ threadId,
835
+ runId: runId(ordinal),
836
+ nodeId: rootNodeId(ordinal),
837
+ driver,
838
+ occurredAt: at(ordinal),
839
+ payload: {
840
+ ...item(ordinal, ordinal * 10 + 1, status),
841
+ id: TurnItemId.make(`item:compacted:compaction-${ordinal}`),
842
+ type: "compaction",
843
+ driver,
844
+ },
845
+ });
846
+ for (const [ordinal, status, endedAt] of [
847
+ [2, "queued", null],
848
+ [3, "completed"],
849
+ // A held queue ran this old run after the compactions.
850
+ [4, "completed", at(11.5)],
851
+ [5, "completed"],
852
+ [6, "completed"],
853
+ [7, "completed"],
854
+ [8, "completed"],
855
+ [9, "completed"],
856
+ [10, "completed"],
857
+ [11, "completed"],
858
+ [12, "completed"],
859
+ [13, "rolled_back"],
860
+ [14, "completed"],
861
+ [15, "running", null],
862
+ ] as const) {
863
+ const text = `Prompt ${ordinal}`;
864
+ const authored = { createdBy: "user", creationSource: "web" } as const;
865
+ const nodeId = rootNodeId(ordinal);
866
+ const ended = endedAt === undefined ? at(ordinal + 0.5) : endedAt;
867
+ yield* store.apply({
868
+ id: EventId.make(`event:compacted:run-${ordinal}`),
869
+ type: "run.created",
870
+ threadId,
871
+ runId: runId(ordinal),
872
+ nodeId,
873
+ driver,
874
+ providerInstanceId,
875
+ occurredAt: at(ordinal),
876
+ payload: {
877
+ id: runId(ordinal),
878
+ threadId,
879
+ ordinal,
880
+ providerInstanceId,
881
+ modelSelection,
882
+ providerThreadId: null,
883
+ userMessageId: messageId(ordinal),
884
+ rootNodeId: nodeId,
885
+ activeAttemptId: attemptId(ordinal),
886
+ status,
887
+ requestedAt: at(ordinal),
888
+ startedAt: status === "queued" ? null : at(ordinal),
889
+ completedAt: ended,
890
+ checkpointId: null,
891
+ contextHandoffId: null,
892
+ },
893
+ });
894
+ yield* store.apply({
895
+ id: EventId.make(`event:compacted:attempt-${ordinal}`),
896
+ type: "run-attempt.created",
897
+ threadId,
898
+ runId: runId(ordinal),
899
+ nodeId,
900
+ driver,
901
+ occurredAt: at(ordinal),
902
+ payload: {
903
+ id: attemptId(ordinal),
904
+ runId: runId(ordinal),
905
+ attemptOrdinal: 1,
906
+ rootNodeId: nodeId,
907
+ providerInstanceId,
908
+ providerThreadId,
909
+ providerTurnId: null,
910
+ reason: "initial",
911
+ status: ended === null ? "running" : "completed",
912
+ startedAt: at(ordinal),
913
+ completedAt: ended,
914
+ },
915
+ });
916
+ yield* store.apply({
917
+ id: EventId.make(`event:compacted:node-${ordinal}`),
918
+ type: "node.updated",
919
+ threadId,
920
+ runId: runId(ordinal),
921
+ nodeId,
922
+ driver,
923
+ occurredAt: at(ordinal),
924
+ payload: {
925
+ id: nodeId,
926
+ threadId,
927
+ runId: runId(ordinal),
928
+ rootNodeId: nodeId,
929
+ parentNodeId: null,
930
+ kind: "root_turn",
931
+ status: ended === null ? "running" : "completed",
932
+ countsForRun: true,
933
+ providerThreadId,
934
+ providerTurnId: null,
935
+ nativeItemRef: null,
936
+ runtimeRequestId: null,
937
+ checkpointScopeId: null,
938
+ startedAt: at(ordinal),
939
+ completedAt: ended,
940
+ },
941
+ });
942
+ yield* store.apply({
943
+ id: EventId.make(`event:compacted:turn-${ordinal}`),
944
+ type: "provider-turn.updated",
945
+ threadId,
946
+ nodeId,
947
+ driver,
948
+ occurredAt: at(ordinal),
949
+ payload: {
950
+ id: ProviderTurnId.make(`turn:compacted:${ordinal}`),
951
+ providerThreadId,
952
+ nodeId,
953
+ runAttemptId: attemptId(ordinal),
954
+ nativeTurnRef: null,
955
+ ordinal,
956
+ status: ended === null ? "running" : "completed",
957
+ startedAt: at(ordinal),
958
+ completedAt: ended,
959
+ },
960
+ });
961
+ yield* store.apply({
962
+ id: EventId.make(`event:compacted:message-${ordinal}`),
963
+ type: "message.updated",
964
+ threadId,
965
+ runId: runId(ordinal),
966
+ occurredAt: at(ordinal),
967
+ payload: {
968
+ ...authored,
969
+ id: messageId(ordinal),
970
+ threadId,
971
+ runId: runId(ordinal),
972
+ nodeId,
973
+ role: "user",
974
+ text,
975
+ attachments: [],
976
+ streaming: false,
977
+ createdAt: at(ordinal),
978
+ updatedAt: at(ordinal),
979
+ },
980
+ });
981
+ yield* store.apply({
982
+ id: EventId.make(`event:compacted:prompt-${ordinal}`),
983
+ type: "turn-item.updated",
984
+ threadId,
985
+ runId: runId(ordinal),
986
+ nodeId,
987
+ driver,
988
+ occurredAt: at(ordinal),
989
+ payload: {
990
+ ...authored,
991
+ ...item(ordinal, ordinal * 10, status === "queued" ? "running" : "completed"),
992
+ ...(status === "queued" ? { status: "pending" as const } : {}),
993
+ id: TurnItemId.make(`item:compacted:prompt-${ordinal}`),
994
+ type: "user_message",
995
+ messageId: messageId(ordinal),
996
+ inputIntent: status === "queued" ? "queued_turn" : "turn_start",
997
+ text,
998
+ attachments: [],
999
+ },
1000
+ });
1001
+ }
1002
+ // Each collection as the ordinals of the runs it was read for. Run 1 is the helper's.
1003
+ const ordinals = (ids: ReadonlyArray<string | null>) =>
1004
+ [...new Set(ids.map((id) => Number(id?.split(":").at(-1)) || 1))].toSorted((a, b) => a - b);
1005
+ const read = (keepRunIds: ReadonlyArray<RunId> = [], keepMessageId = messageId(0)) =>
1006
+ store
1007
+ .getThreadRecords(
1008
+ threadId,
1009
+ ["runs", "attempts", "nodes", "providerTurns", "messages", "turnItems"],
1010
+ {
1011
+ messageRoles: ["user"],
1012
+ omitSettledLeafNodes: true,
1013
+ sinceLastCompaction: { keepRunIds, keepMessageId },
1014
+ },
1015
+ )
1016
+ .pipe(
1017
+ Effect.map((records) => ({
1018
+ runs: records.runs.length,
1019
+ attempts: ordinals(records.attempts.map((attempt) => attempt.runId)),
1020
+ nodes: ordinals(records.nodes.map((node) => node.runId)),
1021
+ providerTurns: ordinals(records.providerTurns.map((turn) => turn.runAttemptId)),
1022
+ messages: ordinals(records.messages.map((message) => message.runId)),
1023
+ turnItems: ordinals(records.turnItems.map((row) => row.runId)),
1024
+ })),
1025
+ );
1026
+ const all = [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15];
1027
+ // Every run and every turn item is read, so a model switch is still measured from the last
1028
+ // prompt. Run 1, the helper's, holds only a command item.
1029
+ const expected = (recent: ReadonlyArray<number>) => ({
1030
+ runs: 15,
1031
+ attempts: recent,
1032
+ nodes: recent,
1033
+ providerTurns: recent,
1034
+ messages: recent,
1035
+ turnItems: [1, ...all],
1036
+ });
1037
+
1038
+ // Nothing is compacted yet, so the whole history is current.
1039
+ assert.deepEqual(yield* read(), expected(all));
1040
+
1041
+ // The window starts at the compaction's run. Queued run 2 and held run 4 are older but had
1042
+ // not ended by then.
1043
+ yield* compaction(7, "completed");
1044
+ assert.deepEqual(yield* read(), expected([2, 4, 7, 8, 9, 10, 11, 12, 13, 14, 15]));
1045
+
1046
+ // The latest compaction wins.
1047
+ yield* compaction(11, "completed");
1048
+ const latest = expected([2, 4, 11, 12, 13, 14, 15]);
1049
+ assert.deepEqual(yield* read(), latest);
1050
+
1051
+ // A compaction that was rolled back, failed or is still running does not move the window.
1052
+ yield* compaction(13, "completed");
1053
+ yield* compaction(14, "failed");
1054
+ yield* compaction(15, "running");
1055
+ assert.deepEqual(yield* read(), latest);
1056
+
1057
+ // A named run and the run that holds a re-sent message are read too.
1058
+ assert.deepEqual(
1059
+ yield* read([runId(3)], messageId(6)),
1060
+ expected([2, 3, 4, 6, 11, 12, 13, 14, 15]),
1061
+ );
1062
+
1063
+ // A command also skips the node and the provider thread of each settled task. Tasks ended on
1064
+ // both sides of the compaction; one ran in the provider thread the thread's own runs use.
1065
+ const addProviderThread = (id: ProviderThreadId, appThreadId: ThreadId) =>
1066
+ store.apply({
1067
+ id: EventId.make(`event:compacted:${id}`),
1068
+ type: "provider-thread.updated",
1069
+ threadId,
1070
+ occurredAt: created,
1071
+ payload: {
1072
+ id,
1073
+ appThreadId,
1074
+ driver,
1075
+ providerInstanceId,
1076
+ providerSessionId: null,
1077
+ ownerNodeId: null,
1078
+ nativeThreadRef: null,
1079
+ nativeConversationHeadRef: null,
1080
+ lastRunOrdinal: null,
1081
+ handoffIds: [],
1082
+ forkedFrom: null,
1083
+ status: "idle",
1084
+ firstRunOrdinal: null,
1085
+ createdAt: created,
1086
+ updatedAt: created,
1087
+ },
1088
+ });
1089
+ yield* addProviderThread(providerThreadId, threadId);
1090
+ for (const [name, ordinal, delivery] of [
1091
+ ["old-delivered", 5, "delivered"],
1092
+ ["old-pending", 5, "pending"],
1093
+ ["old-own-thread", 5, "delivered"],
1094
+ ["recent-delivered", 12, "delivered"],
1095
+ ["recent-pending", 12, "pending"],
1096
+ ] as const) {
1097
+ const id = NodeId.make(`node:compacted:task:${name}`);
1098
+ const taskProviderThreadId =
1099
+ name === "old-own-thread"
1100
+ ? providerThreadId
1101
+ : ProviderThreadId.make(`provider-thread:compacted:task:${name}`);
1102
+ if (taskProviderThreadId !== providerThreadId) {
1103
+ yield* addProviderThread(
1104
+ taskProviderThreadId,
1105
+ ThreadId.make(`thread:compacted:child:${name}`),
1106
+ );
1107
+ }
1108
+ const task = {
1109
+ id,
1110
+ threadId,
1111
+ runId: runId(ordinal),
1112
+ parentNodeId: rootNodeId(ordinal),
1113
+ status: "completed",
1114
+ startedAt: at(ordinal),
1115
+ completedAt: at(ordinal),
1116
+ } as const;
1117
+ yield* store.apply({
1118
+ id: EventId.make(`event:compacted:task:${name}`),
1119
+ type: "subagent.updated",
1120
+ threadId,
1121
+ runId: runId(ordinal),
1122
+ nodeId: id,
1123
+ driver,
1124
+ providerInstanceId,
1125
+ occurredAt: at(ordinal),
1126
+ payload: {
1127
+ ...task,
1128
+ origin: "app_owned",
1129
+ createdBy: "agent",
1130
+ driver,
1131
+ providerInstanceId,
1132
+ providerThreadId: taskProviderThreadId,
1133
+ childThreadId: null,
1134
+ nativeTaskRef: null,
1135
+ prompt: "Do the work",
1136
+ title: name,
1137
+ model: null,
1138
+ completionDelivery: { state: delivery, observedByRunId: null },
1139
+ result: null,
1140
+ updatedAt: at(ordinal),
1141
+ },
1142
+ });
1143
+ yield* store.apply({
1144
+ id: EventId.make(`event:compacted:task-node:${name}`),
1145
+ type: "node.updated",
1146
+ threadId,
1147
+ runId: runId(ordinal),
1148
+ nodeId: id,
1149
+ driver,
1150
+ occurredAt: at(ordinal),
1151
+ payload: {
1152
+ ...task,
1153
+ rootNodeId: rootNodeId(ordinal),
1154
+ kind: "subagent",
1155
+ countsForRun: false,
1156
+ providerThreadId: null,
1157
+ providerTurnId: null,
1158
+ nativeItemRef: null,
1159
+ runtimeRequestId: null,
1160
+ checkpointScopeId: null,
1161
+ },
1162
+ });
1163
+ }
1164
+ const names = (ids: ReadonlyArray<string>) => ids.map((id) => id.split(":").at(-1)).toSorted();
1165
+ const readWithTasks = (filter: ProjectionStore.ProjectionRecordFilter) =>
1166
+ store
1167
+ .getThreadRecords(
1168
+ threadId,
1169
+ [
1170
+ "runs",
1171
+ "attempts",
1172
+ "nodes",
1173
+ "providerThreads",
1174
+ "providerTurns",
1175
+ "messages",
1176
+ "turnItems",
1177
+ ],
1178
+ { messageRoles: ["user"], omitSettledLeafNodes: true, ...filter },
1179
+ )
1180
+ .pipe(
1181
+ Effect.map((records) => ({
1182
+ runs: names(records.runs.map((run) => run.id)),
1183
+ turnItems: names(records.turnItems.map((row) => row.id)),
1184
+ attempts: names(records.attempts.map((attempt) => attempt.id)),
1185
+ rootNodes: names(
1186
+ records.nodes.flatMap((node) => (node.kind === "root_turn" ? [node.id] : [])),
1187
+ ),
1188
+ taskNodes: names(
1189
+ records.nodes.flatMap((node) => (node.kind === "subagent" ? [node.id] : [])),
1190
+ ),
1191
+ providerThreads: names(records.providerThreads.map((thread) => thread.id)),
1192
+ providerTurns: names(records.providerTurns.map((turn) => turn.id)),
1193
+ messages: names(records.messages.map((message) => message.id)),
1194
+ })),
1195
+ );
1196
+ const window = {
1197
+ sinceLastCompaction: { keepRunIds: [runId(3)], keepMessageId: messageId(6) },
1198
+ };
1199
+ const windowed = yield* readWithTasks(window);
1200
+ const unsettled = yield* readWithTasks({ omitSettledSubagentRecords: true });
1201
+ const both = yield* readWithTasks({ ...window, omitSettledSubagentRecords: true });
1202
+ // Each filter alone keeps records the other one drops.
1203
+ assert.deepEqual(windowed.taskNodes, ["recent-delivered", "recent-pending"]);
1204
+ assert.lengthOf(windowed.providerThreads, 5);
1205
+ assert.deepEqual(unsettled.taskNodes, ["old-pending", "recent-pending"]);
1206
+ assert.deepEqual(unsettled.providerThreads, ["compacted", "old-pending", "recent-pending"]);
1207
+ assert.lengthOf(unsettled.rootNodes, all.length);
1208
+ assert.lengthOf(unsettled.runs, 15);
1209
+ // Together: every run, every turn item and the provider thread the runs use stay, and the
1210
+ // windowed records are read for the recent and named runs, less each settled task's node.
1211
+ const recentRuns = ["11", "12", "13", "14", "15", "2", "3", "4", "6"];
1212
+ assert.deepEqual(both, {
1213
+ ...unsettled,
1214
+ attempts: recentRuns,
1215
+ rootNodes: recentRuns,
1216
+ taskNodes: ["recent-pending"],
1217
+ providerTurns: recentRuns,
1218
+ messages: recentRuns,
1219
+ });
1220
+ const keptUnsettled = new Map(Object.entries(unsettled));
1221
+ assert.deepEqual(
1222
+ Object.fromEntries(
1223
+ Object.entries(windowed).map(([key, kept]) => [
1224
+ key,
1225
+ kept.filter((name) => keptUnsettled.get(key)?.includes(name)),
1226
+ ]),
1227
+ ),
1228
+ both,
1229
+ );
1230
+ }).pipe(Effect.provide(layerTest)),
1231
+);
1232
+
1233
it.effect("record reads filter notification sources without dropping matching message bodies", () =>
1234
Effect.gen(function* () {
1235
const store = yield* ProjectionStore.ProjectionStoreV2;
apps/server/src/orchestration-v2/ProjectionStore.ts
+82
-3
@@ -378,6 +378,16 @@ export interface ProjectionRecordFilter {
378
* the thread stays.
379
*/
380
readonly omitSettledSubagentRecords?: boolean;
381
+ /**
382
+ * For a compacted thread: reads attempts, nodes, messages and provider turns only for runs from
383
+ * the latest completed compaction's run on, runs that had not ended when that run was
384
+ * requested, and the runs named here. Runs and turn items are not narrowed.
385
+ */
386
+ readonly sinceLastCompaction?: {
387
+ readonly keepRunIds: ReadonlyArray<RunId>;
388
+ /** Keeps the run that already holds this message, for a message that is sent again. */
389
+ readonly keepMessageId: MessageId;
390
+ };
391
}
392
393
// A fork inherits its source's history before it has messages of its own,
@@ -3435,6 +3445,49 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
3445
const threadColumn = (ids: ReadonlyArray<unknown> | undefined, column = "thread_id") =>
3446
sql.literal(ids === undefined ? column : `+${column}`);
3447
3448
+ /** The runs whose records a new message reads. Null when nothing was compacted. */
3449
+ const runIdsSinceLastCompaction = Effect.fn("ProjectionStore.runIdsSinceLastCompaction")(
3450
+ function* (
3451
+ threadId: ThreadId,
3452
+ keep: NonNullable<ProjectionRecordFilter["sinceLastCompaction"]>,
3453
+ ) {
3454
+ // A held queue can run out of ordinal order, so a run that ended after the compaction's
3455
+ // run was requested is recent too. Every run row is read anyway, so this scan reads no
3456
+ // extra pages.
3457
+ const recent = yield* sql<{ readonly run_id: RunId }>`
3458
+ WITH compaction AS (
3459
+ SELECT run.ordinal, run.requested_at
3460
+ FROM orchestration_v2_projection_turn_items AS item
3461
+ INDEXED BY orchestration_v2_projection_turn_items_conversation_idx
3462
+ INNER JOIN orchestration_v2_projection_runs AS run ON run.run_id = item.run_id
3463
+ WHERE item.thread_id = ${threadId}
3464
+ AND item.type = 'compaction' AND item.status = 'completed'
3465
+ AND item.type NOT IN ${CONVERSATION_INDEX_EXCLUDED_TYPES}
3466
+ AND run.status <> 'rolled_back'
3467
+ ORDER BY item.ordinal DESC
3468
+ LIMIT 1
3469
+ )
3470
+ SELECT run.run_id
3471
+ FROM compaction
3472
+ INNER JOIN orchestration_v2_projection_runs AS run ON run.thread_id = ${threadId}
3473
+ WHERE run.ordinal >= compaction.ordinal
3474
+ OR run.completed_at IS NULL
3475
+ OR run.completed_at >= compaction.requested_at
3476
+ OR run.status IN ('queued', 'preparing', 'starting', 'running', 'waiting')
3477
+ UNION
3478
+ SELECT message.run_id
3479
+ FROM compaction
3480
+ INNER JOIN orchestration_v2_projection_messages AS message
3481
+ ON message.message_id = ${keep.keepMessageId}
3482
+ WHERE message.thread_id = ${threadId} AND message.run_id IS NOT NULL
3483
+ `;
3484
+ // The compaction's own run is always recent, so no rows means no compaction.
3485
+ return recent.length === 0
3486
+ ? null
3487
+ : encodeIdList([...recent.map((row) => row.run_id), ...keep.keepRunIds]);
3488
+ },
3489
+ );
3490
+
3491
const readCanonicalProjection = Effect.fn("ProjectionStore.readCanonicalProjection")(
3492
function* (
3493
threadId: ThreadId,
@@ -3452,6 +3505,19 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
3505
if (!threadRow) {
3506
return yield* new ProjectionStoreThreadNotFoundError({ threadId });
3507
}
3508
+ const recentRunIds =
3509
+ window === undefined && filter?.sinceLastCompaction !== undefined
3510
+ ? yield* runIdsSinceLastCompaction(threadId, filter.sinceLastCompaction)
3511
+ : null;
3512
+ const inRecentRuns = (column: string) =>
3513
+ recentRunIds === null
3514
+ ? sql``
3515
+ : sql`AND ${sql.literal(column)} IN (SELECT value FROM json_each(${recentRunIds}))`;
3516
+ // Without the hint SQLite reads every row of the thread to find the recent runs' rows.
3517
+ const recentRunIndex = (index: string) =>
3518
+ recentRunIds === null
3519
+ ? sql``
3520
+ : sql`INDEXED BY ${sql.literal(`orchestration_v2_projection_${index}`)}`;
3521
3522
type WindowPayloadRow = PayloadRow & {
3523
readonly work_last_id?: string | null;
@@ -3729,6 +3795,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
3795
SELECT payload_json
3796
FROM orchestration_v2_projection_run_attempts
3797
WHERE thread_id = ${threadId}
3798
+ ${inRecentRuns("run_id")}
3799
ORDER BY run_id ASC, attempt_ordinal ASC
3800
`
3801
: sql<PayloadRow>`
@@ -3749,13 +3816,15 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
3816
filter?.omitSettledLeafNodes === true
3817
? // Seeks the kept nodes. One filter over the thread reads every settled leaf to
3818
// drop it. The first predicate must match migration 083's index expression. A
3752
- // node's run is always a run of its own thread.
3819
+ // node's run is always a run of its own thread, and an active run is recent.
3820
sql`node.rowid IN (
3821
SELECT rowid FROM orchestration_v2_projection_nodes
3822
+ ${recentRunIndex("nodes_thread_run_idx")}
3823
WHERE thread_id = ${threadId}
3824
AND (kind IN ('tool_call', 'assistant_message', 'reasoning')
3825
AND status IN ('completed', 'interrupted', 'failed', 'cancelled', 'rolled_back')
3826
AND run_id IS NOT NULL) = 0
3827
+ ${inRecentRuns("run_id")}
3828
${
3829
filter.omitSettledSubagentRecords === true
3830
? // A subagent's node shares its id. Both indexes hold every column
@@ -3781,7 +3850,8 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
3850
AND kind IN ('tool_call', 'assistant_message', 'reasoning')
3851
AND status IN ('completed', 'interrupted', 'failed', 'cancelled', 'rolled_back')
3852
)`
3784
- : sql`${threadColumn(filter?.nodeIds, "node.thread_id")} = ${threadId}`
3853
+ : sql`${threadColumn(filter?.nodeIds, "node.thread_id")} = ${threadId}
3854
+ ${inRecentRuns("node.run_id")}`
3855
}
3856
${filter?.nodeIds === undefined ? sql`` : sql`AND node.node_id IN (SELECT value FROM json_each(${encodeIdList(filter.nodeIds)}))`}
3857
ORDER BY COALESCE(node.started_at, ''), node.node_id ASC
@@ -3912,6 +3982,13 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
3982
SELECT payload_json
3983
FROM orchestration_v2_projection_provider_turns
3984
WHERE thread_id = ${threadId}
3985
+ ${
3986
+ recentRunIds === null
3987
+ ? sql``
3988
+ : sql`AND run_attempt_id IN (
3989
+ SELECT attempt_id FROM orchestration_v2_projection_run_attempts
3990
+ WHERE thread_id = ${threadId} ${inRecentRuns("run_id")})`
3991
+ }
3992
${filter?.providerTurnAttemptIds === undefined ? sql`` : sql`AND run_attempt_id IN (SELECT value FROM json_each(${encodeIdList(filter.providerTurnAttemptIds)}))`}
3993
${filter?.providerTurnThreadIds === undefined ? sql`` : sql`AND provider_thread_id IN (SELECT value FROM json_each(${encodeIdList(filter.providerTurnThreadIds)}))`}
3994
${
@@ -3957,7 +4034,9 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
4034
: window === undefined
4035
? sql<PayloadRow>`
4036
SELECT payload_json FROM orchestration_v2_projection_messages
3960
- WHERE ${threadColumn(filter?.messageIds)} = ${threadId}
4037
+ ${recentRunIndex("messages_run_idx")}
4038
+ WHERE ${recentRunIds === null ? threadColumn(filter?.messageIds) : sql.literal("thread_id")} = ${threadId}
4039
+ ${inRecentRuns("run_id")}
4040
${filter?.messageIds === undefined ? sql`` : sql`AND message_id IN (SELECT value FROM json_each(${encodeIdList(filter.messageIds)}))`}
4041
${filter?.messageRoles === undefined ? sql`` : sql`AND role IN (SELECT value FROM json_each(${encodeIdList(filter.messageRoles)}))`}
4042
${filter?.monitorNotificationsOnly === true ? sql`AND json_extract(payload_json, '$.notification.source.kind') = 'monitor'` : sql``}
apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts
+92
-5
@@ -1635,6 +1635,8 @@ const restartCompletionWake = Effect.fn("restartCompletionWake")(function* (
1635
"events" | "started" | "startFirstTurn"
1636
>,
1637
siblingFinishesFirst: boolean,
1638
+ userRunBeforeWake = false,
1639
+ intentNamesTarget = false,
1640
) {
1641
const orchestrator = yield* Orchestrator.OrchestratorV2;
1642
const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
@@ -1736,6 +1738,24 @@ const restartCompletionWake = Effect.fn("restartCompletionWake")(function* (
1738
})),
1739
});
1740
yield* finishTurn(origin);
1741
+ // The wake queues behind a user run that started after the origin run finished.
1742
+ if (userRunBeforeWake) {
1743
+ const userRunning = yield* watchTurnRunning(2);
1744
+ yield* orchestrator.dispatch({
1745
+ type: "message.dispatch",
1746
+ commandId: CommandId.make("command:user-run"),
1747
+ threadId,
1748
+ messageId: MessageId.make("message:user-run"),
1749
+ text: "unrelated question",
1750
+ attachments: [],
1751
+ modelSelection: runSelection,
1752
+ dispatchMode: { type: "start_immediately" },
1753
+ createdBy: "user",
1754
+ creationSource: "web",
1755
+ });
1756
+ yield* worker.drain();
1757
+ yield* Fiber.join(userRunning);
1758
+ }
1759
1760
const completeTask = (taskId: NodeId) =>
1761
orchestrator.dispatch({
@@ -1769,19 +1789,55 @@ const restartCompletionWake = Effect.fn("restartCompletionWake")(function* (
1789
createdBy: "agent",
1790
creationSource: "server",
1791
});
1772
- const wakeRunning = yield* watchTurnRunning(2);
1792
+ const wakeOrdinal = userRunBeforeWake ? 3 : 2;
1793
+ const wakeRunning = yield* watchTurnRunning(wakeOrdinal);
1794
+ if (userRunBeforeWake) yield* finishTurn(harness.started[1]!);
1795
yield* orchestrator.resumeQueuedRuns;
1796
yield* worker.drain();
1797
yield* Fiber.join(wakeRunning);
1776
- const wake = harness.started[1]!;
1798
+ const wake = harness.started[wakeOrdinal - 1]!;
1799
assert.equal(wake.message.messageId, firstDelivery.messageId);
1800
+ if (userRunBeforeWake) {
1801
+ // The wake's run holds a compaction, so the origin run's records are older than the window.
1802
+ const compactedAt = yield* DateTime.now;
1803
+ yield* sink.write({
1804
+ events: [
1805
+ {
1806
+ id: EventId.make("event:wake-compaction"),
1807
+ type: "turn-item.updated" as const,
1808
+ threadId,
1809
+ runId: wake.runId,
1810
+ nodeId: wake.rootNodeId,
1811
+ occurredAt: compactedAt,
1812
+ payload: {
1813
+ id: TurnItemId.make("item:wake-compaction"),
1814
+ threadId,
1815
+ runId: wake.runId,
1816
+ nodeId: wake.rootNodeId,
1817
+ providerThreadId: null,
1818
+ providerTurnId: null,
1819
+ nativeItemRef: null,
1820
+ parentItemId: null,
1821
+ ordinal: 1_000,
1822
+ status: "completed" as const,
1823
+ title: null,
1824
+ startedAt: compactedAt,
1825
+ completedAt: compactedAt,
1826
+ updatedAt: compactedAt,
1827
+ type: "compaction" as const,
1828
+ driver,
1829
+ },
1830
+ },
1831
+ ],
1832
+ });
1833
+ }
1834
1835
if (siblingFinishesFirst) {
1836
yield* completeTask(siblingTaskId);
1837
}
1838
1839
// The provider cannot steer natively, so the steer restarts the wake run.
1784
- const steeredRunning = yield* watchTurnRunning(3);
1840
+ const steeredRunning = yield* watchTurnRunning(wakeOrdinal + 1);
1841
yield* orchestrator.dispatch({
1842
type: "message.dispatch",
1843
commandId: CommandId.make("command:steer-wake"),
@@ -1790,13 +1846,15 @@ const restartCompletionWake = Effect.fn("restartCompletionWake")(function* (
1846
text: "also check the logs",
1847
attachments: [],
1848
modelSelection: runSelection,
1793
- dispatchMode: { type: "steer_active", targetRunId: wake.runId },
1849
+ ...(intentNamesTarget
1850
+ ? { dispatchMode: { type: "start_immediately" as const }, deliveryIntent: "restart" as const }
1851
+ : { dispatchMode: { type: "steer_active" as const, targetRunId: wake.runId } }),
1852
createdBy: "user",
1853
creationSource: "web",
1854
});
1855
yield* worker.drain();
1856
yield* Fiber.join(steeredRunning);
1799
- const steered = harness.started[2]!;
1857
+ const steered = harness.started[wakeOrdinal]!;
1858
assert.equal(steered.runId, wake.runId);
1859
assert.equal(
1860
(yield* orchestrator.getThreadProjection(threadId)).attempts.find(
@@ -1883,6 +1941,35 @@ it.effect("wakes the parent for a sibling result after a steer restarts the comp
1941
),
1942
);
1943
1944
+it.effect.each([false, true])(
1945
+ "settles a steered completion wake whose parent run is before the last compaction (restart by intent: %s)",
1946
+ (intentNamesTarget) =>
1947
+ Effect.scoped(
1948
+ Effect.gen(function* () {
1949
+ const harness = yield* nextTurnSelectionHarness(
1950
+ `steered-wake-after-compaction-${intentNamesTarget}`,
1951
+ true,
1952
+ false,
1953
+ );
1954
+ yield* Effect.gen(function* () {
1955
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
1956
+ const { threadId, steered, firstTaskId, finishTurn } = yield* restartCompletionWake(
1957
+ harness,
1958
+ false,
1959
+ true,
1960
+ intentNamesTarget,
1961
+ );
1962
+ yield* finishTurn(steered);
1963
+ const tasks = (yield* orchestrator.getThreadProjection(threadId)).subagents;
1964
+ assert.equal(
1965
+ tasks.find((task) => task.id === firstTaskId)?.completionDelivery?.state,
1966
+ "delivered",
1967
+ );
1968
+ }).pipe(Effect.provide(harness.layer));
1969
+ }),
1970
+ ),
1971
+);
1972
+
1973
it.effect(
1974
"offers the next completion wake when a steer restarts a wake a sibling finished behind",
1975
() =>