@devries-cameron20 / t3code-1 / commits / 7cf653bca6
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

perf(server): narrow delegated recovery reads to delivery records (#29435)

Yash Singh committed Nov 2, 2026 at 10:39 UTC 7cf653bca60d186e977c0809ed18be20f1076666
4 files changed +204 -18
apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts
+71
@@ -25,6 +25,7 @@ import * as Deferred from "effect/Deferred";
25 import * as Fiber from "effect/Fiber";
26 import * as Layer from "effect/Layer";
27 import * as Stream from "effect/Stream";
28 +import * as Tracer from "effect/Tracer";
29
30 import * as CheckpointStore from "../checkpointing/CheckpointStore.ts";
31 import * as ServerConfig from "../config.ts";
@@ -1346,6 +1347,76 @@ const seedRestartCancelledChild = (input: {
1347 it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))(
1348 "subagent follow-ups",
1349 (it) => {
1350 + it.effect("recovers a reserved delivery without loading conversation history", () =>
1351 + Effect.gen(function* () {
1352 + const orchestrator = yield* Orchestrator.OrchestratorV2;
1353 + const sink = yield* EventSink.EventSinkV2;
1354 + const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests;
1355 + const now = yield* DateTime.now;
1356 + const threadId = ThreadId.make("thread:bounded-delivery-recovery");
1357 + const runId = RunId.make("run:bounded-delivery-recovery");
1358 + const taskId = NodeId.make("task:bounded-delivery-recovery");
1359 + yield* seedParentWithTerminalTask({
1360 + threadId,
1361 + runId,
1362 + taskId,
1363 + projectId: ProjectId.make("project:bounded-delivery-recovery"),
1364 + rootNodeId: NodeId.make("node:bounded-delivery-recovery"),
1365 + deliveryState: "claimed",
1366 + deliveryTaskIds: [taskId],
1367 + completionWake: "always",
1368 + now,
1369 + });
1370 + for (const suffix of ["first", "second"]) {
1371 + yield* sink.write({
1372 + events: [
1373 + {
1374 + id: EventId.make(`event:bounded-delivery-history:${suffix}`),
1375 + type: "message.updated",
1376 + threadId,
1377 + occurredAt: now,
1378 + payload: {
1379 + id: MessageId.make(`message:bounded-delivery-history:${suffix}`),
1380 + threadId,
1381 + runId: null,
1382 + nodeId: null,
1383 + createdBy: "user",
1384 + creationSource: "web",
1385 + role: "user",
1386 + text: "Unrelated historical input",
1387 + attachments: [],
1388 + streaming: false,
1389 + createdAt: now,
1390 + updatedAt: now,
1391 + },
1392 + },
1393 + ],
1394 + });
1395 + }
1396 + const statements: string[] = [];
1397 + const tracer = Tracer.make({
1398 + span(options) {
1399 + const span = new Tracer.NativeSpan(options);
1400 + const end = span.end.bind(span);
1401 + span.end = (time, exit) => {
1402 + end(time, exit);
1403 + const query = span.attributes.get("db.query.text");
1404 + if (typeof query === "string") statements.push(query);
1405 + };
1406 + return span;
1407 + },
1408 + });
1409 + yield* orchestrator.recoverDelegatedTasks.pipe(Effect.withTracer(tracer));
1410 + assert.isFalse(statements.some((query) => query.includes("MIN(ordinal)")));
1411 + const messageReads = statements.filter((query) =>
1412 + query.includes("FROM orchestration_v2_projection_messages"),
1413 + );
1414 + assert.isNotEmpty(messageReads);
1415 + assert.isTrue(messageReads.every((query) => query.includes("message_id IN")));
1416 + const wake = yield* requests.take;
1417 + assert.equal(wake.delegatedCompletion?.parentRunId, runId);
1418 + }),
1419 + );
1420 it.effect("keeps a checkpoint completion committed between finalize's read and write", () =>
1421 Effect.gen(function* () {
1422 const orchestrator = yield* Orchestrator.OrchestratorV2;
apps/server/src/orchestration-v2/Orchestrator.ts
+31 -14
@@ -1346,27 +1346,40 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1346 const offerDelegatedCompletionDelivery =
1347 /* This helper runs for individual orchestration events; the enclosing operation owns the span. */
1348 Effect.fnUntraced(function* (threadId: ThreadId, parentRunId: RunId) {
1349 - const projection = yield* projectionStore.getThreadRecords(
1350 - threadId,
1351 - ["runs", "messages", "providerTurns", "providerThreads"],
1352 - { messageRoles: ["user"] },
1353 - );
1354 - const parentRun = projection.runs.find((candidate) => candidate.id === parentRunId);
1349 + const controls = yield* projectionStore.getThreadRecords(threadId, ["runs"], {
1350 + runIds: [parentRunId],
1351 + });
1352 + const parentRun = controls.runs[0];
1353 const cohort = parentRun?.delegatedCompletion;
1354 const delivery = cohort?.delivery;
1355 if (
1356 parentRun === undefined ||
1357 cohort?.disposition !== "open" ||
1360 - delivery === null ||
1361 - delivery === undefined ||
1358 + delivery == null ||
1359 delivery.taskIds.length === 0 ||
1363 - projection.thread.archivedAt !== null ||
1364 - projection.thread.deletedAt !== null ||
1365 - (completionDeliveryMessage(projection, delivery) !== undefined &&
1366 - !isUndeliveredMailboxSteer(projection, delivery.messageId))
1367 - ) {
1360 + controls.thread.archivedAt !== null ||
1361 + controls.thread.deletedAt !== null
1362 + )
1363 return;
1364 + const messages = yield* projectionStore.getThreadRecords(threadId, ["messages"], {
1365 + messageIds: [delivery.messageId],
1366 + });
1367 + const message = messages.messages[0];
1368 + if (message !== undefined) {
1369 + if (message.delegatedCompletion === undefined || message.runId === null) return;
1370 + const runs = yield* projectionStore.getThreadRecords(threadId, ["runs"], {
1371 + runIds: [message.runId],
1372 + });
1373 + const run = runs.runs[0];
1374 + const turns = yield* projectionStore.getThreadRecords(threadId, ["providerTurns"], {
1375 + providerTurnAttemptIds: run?.activeAttemptId == null ? [] : [run.activeAttemptId],
1376 + });
1377 + if (!isUndeliveredMailboxSteer({ ...messages, ...runs, ...turns }, delivery.messageId))
1378 + return;
1379 }
1380 + const projection = yield* projectionStore.getThreadRecords(threadId, ["providerThreads"], {
1381 + providerThreadIds: parentRun.providerThreadId === null ? [] : [parentRun.providerThreadId],
1382 + });
1383 const providerThread =
1384 parentRun.providerThreadId === null
1385 ? undefined
@@ -12708,7 +12721,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
12721 const projection = yield* projectionStore.getThreadRecords(
12722 threadId,
12723 ["runs", "messages", "subagents"],
12711 - { messageRoles: ["user"] },
12724 + {
12725 + runIds: [runId, message.delegatedCompletion.parentRunId],
12726 + messageIds: [message.id],
12727 + subagentRunIds: [message.delegatedCompletion.parentRunId],
12728 + },
12729 );
12730 const deliveryRun = projection.runs.find((candidate) => candidate.id === runId);
12731 const deliveryMessage =
apps/server/src/orchestration-v2/ProjectionStore.test.ts
+92
@@ -215,6 +215,98 @@ const addRolledBackRecoveryCandidate = Effect.fn("addRolledBackRecoveryCandidate
215 return threadId;
216 });
217
218 +it.effect("record reads restrict provider histories to delivery identities", () =>
219 + Effect.gen(function* () {
220 + const store = yield* ProjectionStore.ProjectionStoreV2;
221 + const threadId = yield* addRolledBackRecoveryCandidate("delivery-identities");
222 + const sql = yield* SqlClient.SqlClient;
223 + const now = yield* DateTime.now;
224 + for (const suffix of ["selected", "history"]) {
225 + const id = ProviderThreadId.make(`provider-thread:${suffix}`);
226 + yield* store.apply({
227 + id: EventId.make(`event:delivery-identities:${suffix}`),
228 + type: "provider-thread.updated",
229 + threadId,
230 + occurredAt: now,
231 + payload: {
232 + id,
233 + appThreadId: threadId,
234 + driver,
235 + providerInstanceId,
236 + providerSessionId: null,
237 + ownerNodeId: null,
238 + nativeThreadRef: null,
239 + nativeConversationHeadRef: null,
240 + lastRunOrdinal: null,
241 + handoffIds: [],
242 + forkedFrom: null,
243 + status: "active",
244 + firstRunOrdinal: 1,
245 + createdAt: now,
246 + updatedAt: now,
247 + },
248 + });
249 + }
250 + for (const [suffix, attempt] of [
251 + ["selected", "current"],
252 + ["history", "old"],
253 + ["runless", null],
254 + ] as const) {
255 + yield* store.apply({
256 + id: EventId.make(`event:delivery-turn:${suffix}`),
257 + type: "provider-turn.updated",
258 + threadId,
259 + occurredAt: now,
260 + payload: {
261 + id: ProviderTurnId.make(`turn:${suffix}`),
262 + providerThreadId: ProviderThreadId.make("provider-thread:selected"),
263 + nodeId: NodeId.make("node:delivery"),
264 + runAttemptId: attempt === null ? null : RunAttemptId.make(attempt),
265 + nativeTurnRef: null,
266 + ordinal: suffix === "selected" ? 1 : suffix === "history" ? 2 : 3,
267 + status: "completed",
268 + startedAt: now,
269 + completedAt: now,
270 + },
271 + });
272 + }
273 + assert.deepEqual(
274 + (yield* store.getThreadRecords(threadId, ["providerTurns"], {
275 + providerTurnAttemptIds: [RunAttemptId.make("current")],
276 + })).providerTurns.map((turn) => turn.id),
277 + ["turn:selected"],
278 + );
279 + assert.deepEqual(
280 + (yield* store.getThreadRecords(threadId, ["providerTurns"], {
281 + providerTurnAttemptIds: [],
282 + })).providerTurns,
283 + [],
284 + );
285 + const selected = yield* store.getThreadRecords(threadId, ["providerThreads"], {
286 + providerThreadIds: [ProviderThreadId.make("provider-thread:selected")],
287 + });
288 + assert.deepEqual(
289 + selected.providerThreads.map((row) => row.id),
290 + ["provider-thread:selected"],
291 + );
292 + assert.deepEqual(
293 + (yield* store.getThreadRecords(threadId, ["providerThreads"], {
294 + providerThreadIds: [],
295 + })).providerThreads,
296 + [],
297 + );
298 + // Unselected history must not be decoded, even if its payload is unreadable.
299 + yield* sql`UPDATE orchestration_v2_projection_provider_threads SET payload_json = '{}'
300 + WHERE provider_thread_id = 'provider-thread:history'`;
301 + assert.lengthOf(
302 + (yield* store.getThreadRecords(threadId, ["providerThreads"], {
303 + providerThreadIds: [ProviderThreadId.make("provider-thread:selected")],
304 + })).providerThreads,
305 + 1,
306 + );
307 + }).pipe(Effect.provide(layerTest)),
308 +);
309 +
310 it.effect("record reads filter notification sources without dropping matching message bodies", () =>
311 Effect.gen(function* () {
312 const store = yield* ProjectionStore.ProjectionStoreV2;
apps/server/src/orchestration-v2/ProjectionStore.ts
+10 -4
@@ -329,6 +329,8 @@ export interface ProjectionRuntimeResponseContext {
329 }
330
331 export interface ProjectionRecordFilter {
332 + readonly providerThreadIds?: ReadonlyArray<ProviderThreadId>;
333 + readonly providerTurnAttemptIds?: ReadonlyArray<RunAttemptId>;
334 readonly messageRoles?: ReadonlyArray<OrchestrationV2ConversationMessage["role"]>;
335 readonly monitorNotificationsOnly?: boolean;
336 readonly turnItemRunId?: RunId;
@@ -3387,13 +3389,14 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
3389 OR threads.provider_thread_id IN (SELECT value FROM json_each(${cohortProviderThreadIds})))
3390 ORDER BY sessions.updated_at ASC, sessions.provider_session_id ASC
3391 `,
3390 - fields !== undefined && !fields.includes("providerThreads")
3392 + (fields !== undefined && !fields.includes("providerThreads")) ||
3393 + (window === undefined && matchesNothing(filter?.providerThreadIds))
3394 ? Effect.succeed([])
3395 : window === undefined
3396 ? sql<PayloadRow>`
3397 SELECT payload_json
3398 FROM orchestration_v2_projection_provider_threads
3396 - WHERE thread_id = ${threadId}
3399 + WHERE (thread_id = ${threadId}
3400 OR owner_node_id IN (
3401 SELECT node_id
3402 FROM orchestration_v2_projection_nodes
@@ -3404,7 +3407,8 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
3407 FROM orchestration_v2_projection_subagents
3408 WHERE thread_id = ${threadId}
3409 AND provider_thread_id IS NOT NULL
3407 - )
3410 + ))
3411 + ${filter?.providerThreadIds === undefined ? sql`` : sql`AND provider_thread_id IN (SELECT value FROM json_each(${encodeIdList(filter.providerThreadIds)}))`}
3412 ORDER BY COALESCE(first_run_ordinal, 0), provider_thread_id ASC
3413 `
3414 : sql<PayloadRow>`
@@ -3416,13 +3420,15 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
3420 AND provider_thread_id = ${latestTodoList?.provider_thread_id ?? null})
3421 ORDER BY COALESCE(first_run_ordinal, 0), provider_thread_id ASC
3422 `,
3419 - fields !== undefined && !fields.includes("providerTurns")
3423 + (fields !== undefined && !fields.includes("providerTurns")) ||
3424 + (window === undefined && matchesNothing(filter?.providerTurnAttemptIds))
3425 ? Effect.succeed([])
3426 : window === undefined
3427 ? sql<PayloadRow>`
3428 SELECT payload_json
3429 FROM orchestration_v2_projection_provider_turns
3430 WHERE thread_id = ${threadId}
3431 + ${filter?.providerTurnAttemptIds === undefined ? sql`` : sql`AND run_attempt_id IN (SELECT value FROM json_each(${encodeIdList(filter.providerTurnAttemptIds)}))`}
3432 ORDER BY provider_thread_id ASC, ordinal ASC
3433 `
3434 : sql<PayloadRow>`