@devries-cameron20 / t3code-1 / commits / 647bd148b3
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): avoid message ordering scans during recovery and settlement (#30582)

Yash Singh committed Nov 3, 2026 at 21:22 UTC 647bd148b3fbd7c8b57a10fd41ae6d5fc7b35cc1
6 files changed +115 -21
apps/server/src/orchestration-v2/Orchestrator.ts
+10 -7
@@ -3331,16 +3331,19 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
3331 }
3332 }
3333 if (command.type === "thread.settle") {
3334 - const projection = yield* loadProjectionForCommand(
3335 - command,
3336 - ["runs", "runtimeRequests", "messages"],
3337 - { turnItemTypes: [], messageRoles: ["user"] },
3338 - );
3334 + const projection = yield* loadProjectionForCommand(command, ["runs", "runtimeRequests"]);
3335 + const queuedMessages = yield* projectionStore
3336 + .getMessagesById(
3337 + command.threadId,
3338 + projection.runs.filter((run) => run.status === "queued").map((run) => run.userMessageId),
3339 + )
3340 + .pipe(mapDispatchError(command));
3341 const automaticMessageIds = new Set(
3340 - projection.messages
3342 + [...queuedMessages.values()]
3343 .filter(
3344 (message) =>
3343 - message.notification !== undefined || message.delegatedCompletion !== undefined,
3345 + message.role === "user" &&
3346 + (message.notification !== undefined || message.delegatedCompletion !== undefined),
3347 )
3348 .map((message) => message.id),
3349 );
apps/server/src/orchestration-v2/ProjectionStore.test.ts
+52
@@ -282,6 +282,27 @@ it.effect("routes full projection reads to readers and retains uncommitted write
282 forkedFrom: { type: "run", threadId: parentId, runId: parent.runs[0]!.id },
283 },
284 });
285 + const messageId = MessageId.make("message:reader-routing");
286 + yield* store.apply({
287 + id: EventId.make("event:reader-routing:message"),
288 + type: "message.updated",
289 + threadId,
290 + occurredAt: parent.thread.createdAt,
291 + payload: {
292 + id: messageId,
293 + threadId,
294 + runId: null,
295 + nodeId: null,
296 + role: "user",
297 + text: "Committed message",
298 + attachments: [],
299 + streaming: false,
300 + createdBy: "user",
301 + creationSource: "web",
302 + createdAt: parent.thread.createdAt,
303 + updatedAt: parent.thread.createdAt,
304 + },
305 + });
306 const { queries, trace } = traceSqlRouting();
307 yield* store.getAgentAwarenessSnapshot().pipe(trace);
308 const projection = yield* store.getThreadProjection(threadId).pipe(trace);
@@ -292,6 +313,10 @@ it.effect("routes full projection reads to readers and retains uncommitted write
313 .map((row) => row.sourceItemId),
314 parent.turnItems.map((item) => item.id),
315 );
316 + assert.equal(
317 + (yield* store.getMessagesById(threadId, [messageId]).pipe(trace)).get(messageId)?.text,
318 + "Committed message",
319 + );
320 assert.isAbove(queries.length, 1);
321 assert.deepEqual([...new Set(queries.map((row) => row.connection))], ["reader"]);
322 const commitRequested = yield* Deferred.make<void>();
@@ -343,6 +368,15 @@ it.effect("routes full projection reads to readers and retains uncommitted write
368 SET payload_json = json_set(payload_json, '$.title', 'Uncommitted title')
369 WHERE thread_id = ${threadId}
370 `;
371 + yield* sql`
372 + UPDATE orchestration_v2_projection_messages
373 + SET payload_json = json_set(payload_json, '$.text', 'Uncommitted message')
374 + WHERE message_id = ${messageId}
375 + `;
376 + assert.equal(
377 + (yield* store.getMessagesById(threadId, [messageId])).get(messageId)?.text,
378 + "Uncommitted message",
379 + );
380 assert.equal(
381 (yield* store.getAgentAwarenessSnapshot()).threads.find(
382 (thread) => thread.id === threadId,
@@ -2638,10 +2672,28 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
2672 );
2673 yield* insertMessageItem("null-id", threadId, -2, null);
2674 yield* insertMessageItem("missing-id", threadId, -3, undefined);
2675 + const absentMessageId = MessageId.make("message:bounded-sql-history:absent");
2676 + const messageReadTrace = traceSqlStatements();
2677 + const selected = yield* projectionStore
2678 + .getMessagesById(threadId, [firstMessageId, lastMessageId, firstMessageId, absentMessageId])
2679 + .pipe(Effect.withTracer(messageReadTrace.tracer));
2680 + assert.isFalse(messageReadTrace.statements.some((query) => query.includes("turn_items")));
2681 + assert.equal(selected.size, 2);
2682 + assert.equal(selected.get(firstMessageId)?.id, firstMessageId);
2683 + assert.equal(selected.get(lastMessageId)?.id, lastMessageId);
2684 + assert.isFalse(selected.has(absentMessageId));
2685 + assert.equal((yield* projectionStore.getMessagesById(threadId, [])).size, 0);
2686 + assert.equal(
2687 + (yield* projectionStore.getMessagesById(ThreadId.make("other-thread"), [firstMessageId]))
2688 + .size,
2689 + 0,
2690 + );
2691 const readMessageIds = (filter?: ProjectionStore.ProjectionRecordFilter) =>
2692 projectionStore
2693 .getThreadRecords(threadId, ["messages"], filter)
2694 .pipe(Effect.map(({ messages }) => messages.map((message) => message.id)));
2695 + const canonical = yield* projectionStore.getThreadRecords(threadId, ["messages"]);
2696 + for (const message of canonical.messages) assert.deepEqual(selected.get(message.id), message);
2697 assert.deepEqual(yield* readMessageIds(), [lastMessageId, firstMessageId]);
2698 assert.deepEqual(yield* readMessageIds({ messageRoles: ["user", "assistant"] }), [
2699 lastMessageId,
apps/server/src/orchestration-v2/ProjectionStore.ts
+25
@@ -473,6 +473,13 @@ export interface ProjectionStoreV2Shape {
473 readonly threadId: ThreadId;
474 readonly itemId: TurnItemId;
475 }) => Effect.Effect<OrchestrationV2TurnItem | null, ProjectionStoreV2Error>;
476 + readonly getMessagesById: (
477 + threadId: ThreadId,
478 + messageIds: ReadonlyArray<MessageId>,
479 + ) => Effect.Effect<
480 + ReadonlyMap<MessageId, OrchestrationV2ConversationMessage>,
481 + ProjectionStoreV2Error
482 + >;
483 readonly getThreadRecords: <K extends ProjectionRecordField>(
484 threadId: ThreadId,
485 fields: ReadonlyArray<K>,
@@ -5563,6 +5570,23 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
5570 Effect.mapError((cause) => new ProjectionStoreReadError({ threadId, cause })),
5571 );
5572
5573 + const getMessagesById = Effect.fn("ProjectionStore.getMessagesById")(
5574 + function* (threadId: ThreadId, messageIds: ReadonlyArray<MessageId>) {
5575 + if (messageIds.length === 0)
5576 + return new Map<MessageId, OrchestrationV2ConversationMessage>();
5577 + const rows = yield* sql<PayloadRow>`
5578 + SELECT message.payload_json
5579 + FROM json_each(${encodeIdList([...new Set(messageIds)])}) AS requested
5580 + CROSS JOIN orchestration_v2_projection_messages AS message
5581 + ON message.message_id = requested.value
5582 + WHERE message.thread_id = ${threadId}
5583 + `;
5584 + const messages = yield* decodeRows(decodeMessagePayload, threadId)(rows);
5585 + return new Map(messages.map((message) => [message.id, message]));
5586 + },
5587 + (effect, threadId) => effect.pipe(Effect.mapError(controlReadError(threadId))),
5588 + );
5589 +
5590 const getThreadRecords: ProjectionStoreV2Shape["getThreadRecords"] = (
5591 threadId,
5592 fields,
@@ -7385,6 +7409,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
7409 listAppOwnedChildThreadIds: readOnReader(listAppOwnedChildThreadIds),
7410 getNextTurnItemOrdinal: readOnReader(getNextTurnItemOrdinal),
7411 getTurnItem: readOnReader(getTurnItem),
7412 + getMessagesById: readOnReader(getMessagesById),
7413 getThreadRecords: (threadId, fields, filter) =>
7414 getThreadRecords(threadId, fields, filter).pipe(NodeSqliteClient.readOnly),
7415 getRuntimeRequest: readOnReader(getRuntimeRequest),
apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.test.ts
+10
@@ -601,6 +601,16 @@ it.effect.each([
601 Layer.mock(ProjectionStore.ProjectionStoreV2)({
602 getRecoveryThreadIds: () => Effect.succeed([threadId]),
603 getRuntimeRecoveryProjection: () => Effect.succeed(projection),
604 + getMessagesById: (requestedThreadId, messageIds) =>
605 + Effect.succeed(
606 + new Map(
607 + projection.messages
608 + .filter(
609 + (message) => requestedThreadId === threadId && messageIds.includes(message.id),
610 + )
611 + .map((message) => [message.id, message]),
612 + ),
613 + ),
614 getThreadRecords: () => Effect.succeed(projection),
615 }),
616 Layer.mock(EventSink.EventSinkV2)({
apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts
+17 -14
@@ -342,20 +342,23 @@ export const make = Effect.gen(function* () {
342 const queuedMessages =
343 pendingRuns.length === 0
344 ? []
345 - : (yield* projections
346 - .getThreadRecords(projection.thread.id, ["messages"], {
347 - messageIds: pendingRuns.map((run) => run.userMessageId),
348 - })
349 - .pipe(
350 - Effect.mapError(
351 - (cause) =>
352 - new ProviderRuntimeRecoveryError({
353 - operation: "read-projections",
354 - threadId: projection.thread.id,
355 - cause,
356 - }),
357 - ),
358 - )).messages;
345 + : [
346 + ...(yield* projections
347 + .getMessagesById(
348 + projection.thread.id,
349 + pendingRuns.map((run) => run.userMessageId),
350 + )
351 + .pipe(
352 + Effect.mapError(
353 + (cause) =>
354 + new ProviderRuntimeRecoveryError({
355 + operation: "read-projections",
356 + threadId: projection.thread.id,
357 + cause,
358 + }),
359 + ),
360 + )).values(),
361 + ];
362 const bufferedMessageIds = new Set(
363 queuedMessages.filter(isBufferedProviderNotification).map((message) => message.id),
364 );
apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts
+1
@@ -241,6 +241,7 @@ it.effect(
241 listAppOwnedChildThreadIds: () => Effect.die("unused listAppOwnedChildThreadIds"),
242 getNextTurnItemOrdinal: () => Effect.die("unused ordinal read"),
243 getTurnItem: () => Effect.die("unused turn item read"),
244 + getMessagesById: () => Effect.die("unused getMessagesById"),
245 getThreadRecords: () => Effect.die("unused record read"),
246 getRuntimeRequest: () => Effect.die("unused getRuntimeRequest"),
247 getRunningTurnContext: () => Effect.die("unused getRunningTurnContext"),