The best way to control Claude Code, Codex and more from any computer or phone

main
ts 3,186 lines 117 KB
Raw
1 import * as NodeSqlite from "node:sqlite";
2
3 import { assert, it } from "@effect/vitest";
4 import {
5 CommandId,
6 MessageId,
7 EventId,
8 NodeId,
9 type OrchestrationV2DomainEvent,
10 PlanId,
11 ProjectId,
12 ProviderDriverKind,
13 ProviderInstanceId,
14 ProviderSessionId,
15 ProviderThreadId,
16 ProviderTurnId,
17 RunAttemptId,
18 RunId,
19 RuntimeRequestId,
20 ThreadId,
21 TurnItemId,
22 } from "@t3tools/contracts";
23 import * as DateTime from "effect/DateTime";
24 import * as Deferred from "effect/Deferred";
25 import * as Effect from "effect/Effect";
26 import * as Exit from "effect/Exit";
27 import * as Fiber from "effect/Fiber";
28 import * as Layer from "effect/Layer";
29 import * as Metric from "effect/Metric";
30 import * as Option from "effect/Option";
31 import * as Result from "effect/Result";
32 import * as Stream from "effect/Stream";
33 import * as TestClock from "effect/testing/TestClock";
34 import * as Tracer from "effect/Tracer";
35 import * as SqlClient from "effect/sql/SqlClient";
36 import * as Statement from "effect/sql/Statement";
37 import {
38 orchestrationCommandDuration,
39 orchestrationCommandsTotal,
40 } from "../observability/Metrics.ts";
41 import * as SqlitePersistence from "../persistence/Sqlite.ts";
42 import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
43 import * as DelegatedTaskQuestionReactor from "./DelegatedTaskQuestionReactor.ts";
44 import * as EffectOutbox from "./EffectOutbox.ts";
45 import * as EventSink from "./EventSink.ts";
46 import * as Orchestrator from "./Orchestrator.ts";
47 import * as ProjectionStore from "./ProjectionStore.ts";
48 import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts";
49 import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts";
50 import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
51
52 declare module "node:sqlite" {
53 interface StatementSync {
54 // Node 26 has this; @types/node 24 does not declare it yet.
55 stat(counter: "vmStep"): number;
56 }
57 }
58
59 const instanceId = ProviderInstanceId.make("codex");
60 const modelSelection = { instanceId, model: "gpt-5.1-codex" };
61 const adapter = {
62 instanceId,
63 driver: ProviderDriverKind.make("codex"),
64 getCapabilities: () => Effect.succeed(CodexProviderCapabilitiesV2),
65 planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" as const }),
66 openSession: () => Effect.die("No provider process needed for metadata controls"),
67 } as ProviderAdapter.ProviderAdapterV2["Service"];
68 const layerDatabase = SqlitePersistence.layerMemory;
69 const layerTest = Layer.mergeAll(
70 layerDatabase,
71 ProjectionStore.layer.pipe(Layer.provide(layerDatabase)),
72 EffectOutbox.layer.pipe(Layer.provide(layerDatabase)),
73 ProviderReplayHarness.layerWithRegistry(
74 { name: "control-reads" },
75 ProviderAdapterRegistry.layerFromAdapters([adapter]),
76 { databaseLayer: layerDatabase, runEffectWorker: false },
77 ),
78 );
79
80 it.effect.each(
81 (["provider.switch", "thread.model-selection.set"] as const).flatMap((type) =>
82 [
83 { active: true, selectionChange: "instance" },
84 { active: false, selectionChange: "instance" },
85 { active: true, selectionChange: "next-turn" },
86 { active: true, selectionChange: "restart" },
87 ].map((scenario) => ({ type, ...scenario })),
88 ),
89 )("$type $selectionChange ($active)", ({ type, active, selectionChange }) =>
90 Effect.gen(function* () {
91 const orchestrator = yield* Orchestrator.OrchestratorV2;
92 const projections = yield* ProjectionStore.ProjectionStoreV2;
93 const outbox = yield* EffectOutbox.EffectOutboxV2;
94 const threadId = ThreadId.make("thread:instance-switch");
95 const now = yield* DateTime.now;
96 const sessionId = ProviderSessionId.make("session:instance-switch");
97 const providerThreadId = ProviderThreadId.make("provider-thread:instance-switch");
98 const runId = RunId.make("run:instance-switch");
99 const nodeId = NodeId.make("node:instance-switch");
100 const attemptId = RunAttemptId.make("attempt:instance-switch");
101 yield* orchestrator.dispatch({
102 type: "thread.create",
103 commandId: CommandId.make("create-instance-switch"),
104 threadId,
105 projectId: ProjectId.make("project:instance-switch"),
106 title: "Instance switch",
107 modelSelection,
108 runtimeMode: "full-access",
109 interactionMode: "default",
110 branch: null,
111 worktreePath: null,
112 createdBy: "user",
113 creationSource: "web",
114 });
115 yield* projections.apply({
116 id: EventId.make("instance-switch:session"),
117 type: "provider-session.attached",
118 threadId,
119 occurredAt: now,
120 payload: {
121 id: sessionId,
122 driver: adapter.driver,
123 providerInstanceId: instanceId,
124 status: "ready",
125 cwd: "/repo",
126 model: modelSelection.model,
127 capabilities: CodexProviderCapabilitiesV2,
128 createdAt: now,
129 updatedAt: now,
130 lastError: null,
131 },
132 });
133 yield* projections.apply({
134 id: EventId.make("instance-switch:provider-thread"),
135 type: "provider-thread.updated",
136 threadId,
137 occurredAt: now,
138 payload: {
139 id: providerThreadId,
140 driver: adapter.driver,
141 providerInstanceId: instanceId,
142 providerSessionId: sessionId,
143 appThreadId: threadId,
144 ownerNodeId: null,
145 nativeThreadRef: {
146 driver: adapter.driver,
147 nativeId: "native:instance-switch",
148 strength: "strong",
149 },
150 nativeConversationHeadRef: null,
151 status: "idle",
152 firstRunOrdinal: 1,
153 lastRunOrdinal: 1,
154 handoffIds: [],
155 forkedFrom: null,
156 createdAt: now,
157 updatedAt: now,
158 },
159 });
160 if (active) {
161 yield* projections.apply({
162 id: EventId.make("instance-switch:run"),
163 type: "run.created",
164 threadId,
165 occurredAt: now,
166 payload: {
167 id: runId,
168 threadId,
169 ordinal: 1,
170 providerInstanceId: instanceId,
171 modelSelection,
172 providerThreadId,
173 userMessageId: MessageId.make("message:instance-switch"),
174 rootNodeId: nodeId,
175 activeAttemptId: attemptId,
176 status: "running",
177 requestedAt: now,
178 startedAt: now,
179 completedAt: null,
180 checkpointId: null,
181 contextHandoffId: null,
182 },
183 });
184 yield* projections.apply({
185 id: EventId.make("instance-switch:turn"),
186 type: "provider-turn.updated",
187 threadId,
188 runId,
189 occurredAt: now,
190 payload: {
191 id: ProviderTurnId.make("provider-turn:instance-switch"),
192 providerThreadId,
193 nodeId,
194 runAttemptId: attemptId,
195 nativeTurnRef: { driver: adapter.driver, nativeId: "native:turn", strength: "strong" },
196 ordinal: 1,
197 status: "running",
198 startedAt: now,
199 completedAt: null,
200 },
201 });
202 }
203 const before = yield* projections.getThreadProjection(threadId);
204 const sequence = yield* orchestrator.getThreadEventSequence(threadId);
205 const commandId = CommandId.make("switch-instance");
206 const target = {
207 instanceId:
208 selectionChange === "instance" ? ProviderInstanceId.make("codex-other") : instanceId,
209 model: selectionChange === "instance" ? modelSelection.model : "gpt-6",
210 };
211 const outcome = yield* orchestrator
212 .dispatch({
213 type,
214 commandId,
215 threadId,
216 modelSelection: target,
217 })
218 .pipe(Effect.result);
219 const after = yield* projections.getThreadProjection(threadId);
220 if (active && selectionChange !== "next-turn") {
221 assert.deepEqual(
222 after,
223 before,
224 "a refused switch must preserve the turn, session and selection",
225 );
226 assert.deepEqual(
227 yield* outbox.listByCommandId(commandId),
228 [],
229 "a refused switch must not release the provider session",
230 );
231 assert.equal(yield* orchestrator.getThreadEventSequence(threadId), sequence);
232 assert.ok(Result.isFailure(outcome));
233 assert.instanceOf(outcome.failure, Orchestrator.OrchestratorProviderSelectionBusyError);
234 } else {
235 assert.ok(Result.isSuccess(outcome));
236 assert.deepEqual(after.thread.modelSelection, target);
237 assert.equal(after.thread.providerInstanceId, target.instanceId);
238 assert.deepEqual(after.runs, before.runs);
239 assert.deepEqual(after.providerTurns, before.providerTurns);
240 }
241 }).pipe(
242 Effect.provide(
243 Layer.mergeAll(
244 layerDatabase,
245 ProjectionStore.layer.pipe(Layer.provide(layerDatabase)),
246 EffectOutbox.layer.pipe(Layer.provide(layerDatabase)),
247 ProviderReplayHarness.layerWithRegistry(
248 { name: "instance-switch" },
249 ProviderAdapterRegistry.layerFromAdapters([
250 {
251 ...adapter,
252 planSelectionTransition: () =>
253 Effect.succeed(
254 selectionChange === "restart"
255 ? { type: "restart_session" as const }
256 : { type: "apply_on_next_turn" as const },
257 ),
258 },
259 { ...adapter, instanceId: ProviderInstanceId.make("codex-other") },
260 ]),
261 { databaseLayer: layerDatabase, runEffectWorker: false },
262 ),
263 ),
264 ),
265 ),
266 );
267
268 it.effect(
269 "records approval feedback and targets only its runtime request without dispatching a parent turn",
270 () =>
271 Effect.gen(function* () {
272 const orchestrator = yield* Orchestrator.OrchestratorV2;
273 const projections = yield* ProjectionStore.ProjectionStoreV2;
274 const outbox = yield* EffectOutbox.EffectOutboxV2;
275 const threadId = ThreadId.make("thread:approval-feedback");
276 const sessionId = ProviderSessionId.make("session:approval-feedback");
277 const now = yield* DateTime.now;
278 yield* orchestrator.dispatch({
279 type: "thread.create",
280 commandId: CommandId.make("create-approval-feedback"),
281 threadId,
282 projectId: ProjectId.make("project:approval-feedback"),
283 title: "Parent",
284 modelSelection,
285 runtimeMode: "full-access",
286 interactionMode: "default",
287 branch: null,
288 worktreePath: null,
289 createdBy: "user",
290 creationSource: "web",
291 });
292 yield* projections.apply({
293 id: EventId.make("session:approval-feedback"),
294 type: "provider-session.attached",
295 threadId,
296 occurredAt: now,
297 payload: {
298 id: sessionId,
299 driver: adapter.driver,
300 providerInstanceId: instanceId,
301 status: "ready",
302 cwd: "/repo",
303 model: modelSelection.model,
304 capabilities: CodexProviderCapabilitiesV2,
305 createdAt: now,
306 updatedAt: now,
307 lastError: null,
308 },
309 });
310 for (const name of ["child", "other"]) {
311 yield* projections.apply({
312 id: EventId.make(`request:${name}`),
313 type: "runtime-request.updated",
314 threadId,
315 occurredAt: now,
316 payload: {
317 id: RuntimeRequestId.make(name),
318 nodeId: NodeId.make(`node:${name}`),
319 providerTurnId: null,
320 nativeRequestRef: null,
321 kind: "command",
322 status: "pending",
323 responseCapability: {
324 type: "live",
325 providerSessionId: sessionId,
326 supportsFeedback: name === "child",
327 },
328 createdAt: now,
329 resolvedAt: null,
330 },
331 });
332 }
333 const commandId = CommandId.make("respond:approval-feedback");
334 const feedback = "Run it in a safer manner.";
335 for (const [index, invalid] of [
336 { requestId: RuntimeRequestId.make("other"), decision: "decline" as const, feedback },
337 { requestId: RuntimeRequestId.make("child"), decision: "accept" as const, feedback },
338 { requestId: RuntimeRequestId.make("child"), decision: "decline" as const, feedback: " " },
339 ].entries()) {
340 const rejectedId = CommandId.make(`invalid-feedback:${index}`);
341 const rejected = yield* orchestrator
342 .dispatch({
343 type: "runtime-request.respond",
344 commandId: rejectedId,
345 threadId,
346 ...invalid,
347 })
348 .pipe(Effect.result);
349 assert.isTrue(Result.isFailure(rejected));
350 assert.deepEqual(yield* outbox.listByCommandId(rejectedId), []);
351 }
352 yield* orchestrator.dispatch({
353 type: "runtime-request.respond",
354 commandId,
355 threadId,
356 requestId: RuntimeRequestId.make("child"),
357 decision: "decline",
358 feedback,
359 });
360 const projection = yield* projections.getThreadProjection(threadId);
361 assert.equal(
362 projection.runtimeRequests.find((request) => request.id === "child")?.feedback,
363 feedback,
364 );
365 assert.equal(
366 projection.runtimeRequests.find((request) => request.id === "other")?.status,
367 "pending",
368 );
369 assert.equal(projection.messages.at(-1)?.text, feedback);
370 const feedbackMessageId = MessageId.make("approval-feedback:child");
371 assert.ok(
372 projection.visibleTurnItems.some(
373 ({ item }) =>
374 item.type === "user_message" &&
375 item.messageId === feedbackMessageId &&
376 item.text === feedback,
377 ),
378 );
379 const history = yield* projections.getTimelinePage(threadId, { limit: 20, view: "messages" });
380 assert.ok(
381 history.items.some(
382 ({ item }) =>
383 item.type === "user_message" &&
384 item.messageId === feedbackMessageId &&
385 item.text === feedback,
386 ),
387 );
388 assert.equal(projection.runs.length, 0);
389 assert.deepEqual(
390 (yield* outbox.listByCommandId(commandId)).map((effect) => effect.request),
391 [
392 {
393 type: "runtime-request.respond",
394 providerSessionId: sessionId,
395 requestId: RuntimeRequestId.make("child"),
396 decision: "decline",
397 feedback,
398 },
399 ],
400 );
401 }).pipe(Effect.provide(layerTest)),
402 );
403
404 it.effect("marks a never-visited thread's first completion unread", () =>
405 Effect.gen(function* () {
406 const orchestrator = yield* Orchestrator.OrchestratorV2;
407 const projections = yield* ProjectionStore.ProjectionStoreV2;
408 const threadId = ThreadId.make("thread:completion-watermark");
409 yield* orchestrator.dispatch({
410 type: "thread.create",
411 commandId: CommandId.make("create-completion-watermark"),
412 threadId,
413 projectId: ProjectId.make("project:completion-watermark"),
414 title: "Background task",
415 modelSelection,
416 runtimeMode: "full-access",
417 interactionMode: "default",
418 branch: null,
419 worktreePath: null,
420 createdBy: "user",
421 creationSource: "web",
422 });
423 assert.isNull((yield* projections.getThreadShell(threadId))?.lastVisitedAt);
424 const dispatch = (id: string) =>
425 orchestrator.dispatch({
426 type: "message.dispatch",
427 commandId: CommandId.make(id),
428 threadId,
429 messageId: MessageId.make(id),
430 text: "Run in the background",
431 attachments: [],
432 dispatchMode: { type: "defer_start" },
433 createdBy: "agent",
434 creationSource: "mcp",
435 });
436 yield* TestClock.adjust("1 second");
437 const dispatchedAt = yield* DateTime.now;
438 yield* dispatch("first-completion-watermark");
439 yield* orchestrator.dispatch({
440 type: "run.interrupt",
441 commandId: CommandId.make("interrupt-completion-watermark"),
442 threadId,
443 runId: (yield* projections.getThreadProjection(threadId)).runs[0]!.id,
444 });
445 // The run ends in the dispatch's millisecond and must still be unread.
446 const ended = yield* projections.getThreadShell(threadId);
447 assert.ok(ended?.lastVisitedAt && ended.latestRunCompletedAt);
448 assert.equal(
449 DateTime.toEpochMillis(ended.lastVisitedAt),
450 DateTime.toEpochMillis(dispatchedAt) - 1,
451 );
452 assert.isAbove(
453 DateTime.toEpochMillis(ended.latestRunCompletedAt),
454 DateTime.toEpochMillis(ended.lastVisitedAt),
455 );
456
457 yield* TestClock.adjust("1 second");
458 yield* dispatch("second-completion-watermark");
459 assert.deepEqual(
460 (yield* projections.getThreadShell(threadId))?.lastVisitedAt,
461 ended.lastVisitedAt,
462 );
463 }).pipe(Effect.provide(layerTest)),
464 );
465
466 const completeAndSnoozeThread = Effect.fn("test.completeAndSnoozeThread")(function* (
467 name: string,
468 snoozeMinutes: number,
469 ) {
470 const orchestrator = yield* Orchestrator.OrchestratorV2;
471 const projections = yield* ProjectionStore.ProjectionStoreV2;
472 const threadId = ThreadId.make(`thread:${name}`);
473 yield* orchestrator.dispatch({
474 type: "thread.create",
475 commandId: CommandId.make(`create-${name}`),
476 threadId,
477 projectId: ProjectId.make(`project:${name}`),
478 title: "Snoozed completion",
479 modelSelection,
480 runtimeMode: "full-access",
481 interactionMode: "default",
482 branch: null,
483 worktreePath: null,
484 createdBy: "user",
485 creationSource: "web",
486 });
487 yield* orchestrator.dispatch({
488 type: "message.dispatch",
489 commandId: CommandId.make(`dispatch-${name}`),
490 threadId,
491 messageId: MessageId.make(`dispatch-${name}`),
492 text: "Run in the background",
493 attachments: [],
494 dispatchMode: { type: "defer_start" },
495 createdBy: "agent",
496 creationSource: "mcp",
497 });
498 yield* orchestrator.dispatch({
499 type: "run.interrupt",
500 commandId: CommandId.make(`interrupt-${name}`),
501 threadId,
502 runId: (yield* projections.getThreadProjection(threadId)).runs[0]!.id,
503 });
504 const completedAt = (yield* projections.getThreadProjection(threadId)).runs.at(-1)?.completedAt;
505 assert.ok(completedAt);
506 yield* TestClock.adjust("1 second");
507 const snoozedUntil = DateTime.add(yield* DateTime.now, { minutes: snoozeMinutes });
508 yield* orchestrator.dispatch({
509 type: "thread.snooze",
510 commandId: CommandId.make(`snooze-${name}`),
511 threadId,
512 snoozedUntil: DateTime.formatIso(snoozedUntil),
513 });
514 return { threadId, completedAt, snoozedUntil };
515 });
516
517 it.effect("mark unread retires an expired snooze so a dismissed wake stays dismissed", () =>
518 Effect.gen(function* () {
519 const orchestrator = yield* Orchestrator.OrchestratorV2;
520 const projections = yield* ProjectionStore.ProjectionStoreV2;
521 const { threadId, completedAt, snoozedUntil } = yield* completeAndSnoozeThread(
522 "mark-unread-expired-snooze",
523 1,
524 );
525 yield* TestClock.adjust("2 minutes");
526 yield* orchestrator.dispatch({
527 type: "thread.visit",
528 commandId: CommandId.make("dismiss-woke"),
529 threadId,
530 visitedAt: DateTime.formatIso(snoozedUntil),
531 });
532 const dismissed = yield* projections.getThread(threadId);
533 assert.deepEqual(dismissed.lastVisitedAt, snoozedUntil);
534
535 yield* orchestrator.dispatch({
536 type: "thread.mark-unread",
537 commandId: CommandId.make("mark-unread-expired-snooze"),
538 threadId,
539 });
540 assert.deepEqual(yield* projections.getThread(threadId), {
541 ...dismissed,
542 lastVisitedAt: DateTime.subtract(completedAt, { milliseconds: 1 }),
543 snoozedUntil: null,
544 snoozedAt: null,
545 lastSnoozeWakeAt: snoozedUntil,
546 });
547 }).pipe(Effect.provide(layerTest)),
548 );
549
550 it.effect("mark unread leaves a future snooze untouched", () =>
551 Effect.gen(function* () {
552 const orchestrator = yield* Orchestrator.OrchestratorV2;
553 const projections = yield* ProjectionStore.ProjectionStoreV2;
554 const { threadId, completedAt } = yield* completeAndSnoozeThread(
555 "mark-unread-future-snooze",
556 60,
557 );
558 yield* TestClock.adjust("1 minute");
559 yield* orchestrator.dispatch({
560 type: "thread.visit",
561 commandId: CommandId.make("visit-snoozed"),
562 threadId,
563 visitedAt: DateTime.formatIso(yield* DateTime.now),
564 });
565 const visited = yield* projections.getThread(threadId);
566 assert.isNotNull(visited.snoozedUntil);
567
568 yield* orchestrator.dispatch({
569 type: "thread.mark-unread",
570 commandId: CommandId.make("mark-unread-future-snooze"),
571 threadId,
572 });
573 assert.deepEqual(yield* projections.getThread(threadId), {
574 ...visited,
575 lastVisitedAt: DateTime.subtract(completedAt, { milliseconds: 1 }),
576 });
577 }).pipe(Effect.provide(layerTest)),
578 );
579
580 it.effect(
581 "dispatches metadata, queue resume and request controls without hydrating unrelated history",
582 () =>
583 Effect.gen(function* () {
584 const orchestrator = yield* Orchestrator.OrchestratorV2;
585 const projections = yield* ProjectionStore.ProjectionStoreV2;
586 const sql = yield* SqlClient.SqlClient;
587 const outbox = yield* EffectOutbox.EffectOutboxV2;
588 const threadId = ThreadId.make("thread:control-dispatch");
589 const now = yield* DateTime.now;
590 yield* orchestrator.dispatch({
591 type: "thread.create",
592 commandId: CommandId.make("create-control"),
593 threadId,
594 projectId: ProjectId.make("project:control-dispatch"),
595 title: "Before",
596 modelSelection,
597 runtimeMode: "full-access",
598 interactionMode: "default",
599 branch: null,
600 worktreePath: null,
601 createdBy: "user",
602 creationSource: "web",
603 });
604 for (const enabled of [false, true]) {
605 yield* orchestrator.dispatch({
606 type: "thread.auto-settle.set",
607 commandId: CommandId.make(`auto-settle-${enabled}`),
608 threadId,
609 enabled,
610 });
611 const updated = yield* projections.getThreadProjection(threadId);
612 assert.equal(updated.thread.autoSettleDisabledAt == null, enabled);
613 const shell = yield* projections.getThreadShell(threadId);
614 assert.ok(shell);
615 assert.equal(shell.autoSettleDisabledAt == null, enabled);
616 }
617 yield* sql`INSERT INTO orchestration_v2_projection_messages
618 (message_id, thread_id, run_id, node_id, role, streaming, created_at, updated_at, payload_json)
619 VALUES ('obsolete', ${threadId}, NULL, NULL, 'assistant', 0, ${DateTime.formatIso(now)}, ${DateTime.formatIso(now)}, '{"obsolete":true}')`;
620 assert.equal((yield* Effect.exit(projections.getThreadProjection(threadId)))._tag, "Failure");
621 yield* orchestrator.dispatch({
622 type: "queue.resume",
623 commandId: CommandId.make("resume-empty-queue"),
624 threadId,
625 });
626 yield* orchestrator.dispatch({
627 type: "thread.metadata.update",
628 commandId: CommandId.make("rename-control"),
629 threadId,
630 title: "After",
631 });
632 yield* orchestrator.dispatch({
633 type: "thread.runtime-mode.set",
634 commandId: CommandId.make("mode-control"),
635 threadId,
636 runtimeMode: "approval-required",
637 });
638 yield* orchestrator.dispatch({
639 type: "thread.model-selection.set",
640 commandId: CommandId.make("model-control"),
641 threadId,
642 modelSelection: { ...modelSelection, model: "gpt-6" },
643 });
644 const sessionId = ProviderSessionId.make("session:control-dispatch");
645 yield* projections.apply({
646 id: EventId.make("attach-control"),
647 type: "provider-session.attached",
648 threadId,
649 occurredAt: now,
650 payload: {
651 id: sessionId,
652 driver: adapter.driver,
653 providerInstanceId: instanceId,
654 status: "ready",
655 cwd: "/repo",
656 model: "gpt-6",
657 capabilities: CodexProviderCapabilitiesV2,
658 createdAt: now,
659 updatedAt: now,
660 lastError: null,
661 },
662 });
663 for (const mode of ["live", "message"] as const) {
664 const requestId = RuntimeRequestId.make(`request:${mode}`);
665 yield* projections.apply({
666 id: EventId.make(`request:${mode}`),
667 type: "runtime-request.updated",
668 threadId,
669 occurredAt: now,
670 payload: {
671 id: requestId,
672 nodeId: NodeId.make(`node:${mode}`),
673 providerTurnId: null,
674 nativeRequestRef: null,
675 kind: "user_input",
676 status: "pending",
677 responseCapability:
678 mode === "live"
679 ? { type: "live", providerSessionId: sessionId }
680 : { type: "message" },
681 createdAt: now,
682 resolvedAt: null,
683 },
684 });
685 const nodeId = NodeId.make(`node:${mode}`);
686 yield* projections.apply({
687 id: EventId.make(`node:${mode}`),
688 type: "node.updated",
689 threadId,
690 occurredAt: now,
691 payload: {
692 id: nodeId,
693 threadId,
694 runId: null,
695 parentNodeId: null,
696 rootNodeId: nodeId,
697 kind: "user_input_request",
698 status: "waiting",
699 countsForRun: false,
700 providerThreadId: null,
701 providerTurnId: null,
702 nativeItemRef: null,
703 runtimeRequestId: requestId,
704 checkpointScopeId: null,
705 startedAt: now,
706 completedAt: null,
707 },
708 });
709 yield* projections.apply({
710 id: EventId.make(`item:${mode}`),
711 type: "turn-item.updated",
712 threadId,
713 occurredAt: now,
714 payload: {
715 id: TurnItemId.make(`item:${mode}`),
716 threadId,
717 runId: null,
718 nodeId,
719 providerThreadId: null,
720 providerTurnId: null,
721 nativeItemRef: null,
722 parentItemId: null,
723 ordinal: mode === "live" ? 1 : 2,
724 status: "waiting",
725 title: null,
726 startedAt: now,
727 completedAt: null,
728 updatedAt: now,
729 type: "user_input_request",
730 requestId,
731 questions: [],
732 },
733 });
734 const answers = { text: "Blue", list: ["Red"], codex: { answers: ["Green"] } };
735 const normalizedAnswers = { ...answers, codex: ["Green"] };
736 if (mode === "live") {
737 const pending = yield* projections.getRuntimeResponseContext(threadId, requestId);
738 const sequence = yield* orchestrator.getThreadEventSequence(threadId);
739 for (const [index, value] of [
740 null,
741 42,
742 { choice: "Blue" },
743 ["Blue", 42],
744 { answers: [42] },
745 ].entries()) {
746 const commandId = CommandId.make(`invalid-answer:${index}`);
747 const result = yield* orchestrator
748 .dispatch({
749 type: "runtime-request.respond",
750 commandId,
751 threadId,
752 requestId,
753 answers: { ...answers, invalid: value },
754 })
755 .pipe(Effect.result);
756 assert.isTrue(Result.isFailure(result));
757 assert.deepEqual(
758 yield* projections.getRuntimeResponseContext(threadId, requestId),
759 pending,
760 );
761 assert.equal(yield* orchestrator.getThreadEventSequence(threadId), sequence);
762 assert.deepEqual(yield* outbox.listByCommandId(commandId), []);
763 }
764 }
765 if (mode === "live") {
766 yield* orchestrator.dispatch({
767 type: "thread.snooze",
768 commandId: CommandId.make("snooze-live-control"),
769 threadId,
770 snoozedUntil: "2099-07-25T09:00:00.000Z",
771 });
772 }
773 yield* orchestrator.dispatch(
774 mode === "live"
775 ? {
776 type: "runtime-request.respond",
777 commandId: CommandId.make(`respond:${mode}`),
778 threadId,
779 requestId,
780 answers,
781 }
782 : {
783 type: "thread.user-input.dismiss",
784 commandId: CommandId.make(`respond:${mode}`),
785 threadId,
786 requestId,
787 },
788 );
789 assert.equal(
790 (yield* projections.getRuntimeRequest(threadId, requestId))?.status,
791 "resolved",
792 );
793 const response = yield* projections.getRuntimeResponseContext(threadId, requestId);
794 assert.equal(response.node?.status, mode === "live" ? "completed" : "cancelled");
795 assert.equal(response.item?.status, mode === "live" ? "completed" : "cancelled");
796 if (mode === "live") {
797 assert.isNull((yield* projections.getThread(threadId)).snoozedUntil);
798 assert.isNull((yield* projections.getThread(threadId)).snoozedAt);
799 assert.deepEqual(response.request?.answers, normalizedAnswers);
800 assert.equal(response.item?.type, "user_input_request");
801 if (response.item?.type === "user_input_request") {
802 assert.deepEqual(response.item.questionAnswer?.answers, normalizedAnswers);
803 }
804 const effects = yield* outbox.listByCommandId(CommandId.make(`respond:${mode}`));
805 assert.deepEqual(
806 effects.map((effect) => effect.request),
807 [
808 {
809 type: "runtime-request.respond",
810 providerSessionId: sessionId,
811 requestId,
812 answers: normalizedAnswers,
813 },
814 ],
815 );
816 }
817 }
818 yield* orchestrator.dispatch({
819 type: "thread.metadata.update",
820 commandId: CommandId.make("workspace-control"),
821 threadId,
822 worktreePath: "/new-repo",
823 });
824 const thread = yield* projections.getThread(threadId);
825 assert.equal(thread.title, "After");
826 assert.equal(thread.modelSelection.model, "gpt-6");
827 assert.equal(thread.runtimeMode, "approval-required");
828 assert.deepEqual(
829 (yield* projections.getThreadProviderContext(threadId)).providerSessions,
830 [],
831 );
832 yield* sql`INSERT INTO orchestration_v2_projection_turn_items
833 (turn_item_id, thread_id, run_id, node_id, provider_thread_id, provider_turn_id,
834 type, status, ordinal, updated_at, payload_json)
835 VALUES ('obsolete-output', ${threadId}, NULL, NULL, NULL, NULL,
836 'command_execution', 'completed', 900, ${DateTime.formatIso(now)}, '{"obsolete":true}')`;
837 yield* sql`INSERT INTO orchestration_v2_projection_plans
838 (plan_id, thread_id, run_id, node_id, kind, status, payload_json)
839 VALUES ('obsolete-plan', ${threadId}, NULL, 'old-node', 'proposed', 'completed', '{"obsolete":true}')`;
840 yield* sql`INSERT INTO orchestration_v2_projection_context_handoffs
841 (context_handoff_id, thread_id, target_run_id, to_provider_thread_id, strategy, status, updated_at, payload_json)
842 VALUES ('obsolete-handoff', ${threadId}, 'old-run', 'old-provider-thread', 'full_thread_summary', 'ready', ${DateTime.formatIso(now)}, '{"obsolete":true}')`;
843 for (const status of [
844 "resolved_native",
845 "resolved_portable",
846 "failed",
847 "consumed",
848 "superseded",
849 ]) {
850 yield* sql`INSERT INTO orchestration_v2_projection_context_transfers
851 (context_transfer_id, source_thread_id, target_thread_id, target_run_id,
852 type, status, updated_at, payload_json)
853 VALUES (${`obsolete-transfer:${status}`}, 'old-source', ${threadId}, NULL,
854 'merge_back', ${status}, ${DateTime.formatIso(now)}, '{"obsolete":true}')`;
855 }
856 assert.equal(
857 (yield* Effect.exit(projections.getThreadRecords(threadId, ["contextTransfers"])))._tag,
858 "Failure",
859 );
860 // A delegated task that ended and handed its result over.
861 yield* sql`INSERT INTO orchestration_v2_projection_subagents
862 (subagent_id, thread_id, run_id, parent_node_id, provider, provider_thread_id,
863 child_thread_id, origin, status, updated_at, payload_json, completion_delivery_state)
864 VALUES ('obsolete-task', ${threadId}, 'old-run', 'old-node', 'codex', 'obsolete-task-thread',
865 'old-child', 'app_owned', 'completed', ${DateTime.formatIso(now)}, '{"obsolete":true}',
866 'delivered')`;
867 yield* sql`INSERT INTO orchestration_v2_projection_nodes
868 (node_id, thread_id, run_id, root_node_id, kind, status, payload_json)
869 VALUES ('obsolete-task', ${threadId}, 'old-run', 'old-node', 'subagent', 'completed',
870 '{"obsolete":true}')`;
871 yield* sql`INSERT INTO orchestration_v2_projection_provider_threads
872 (provider_thread_id, thread_id, provider, status, updated_at, payload_json)
873 VALUES ('obsolete-task-thread', 'old-child', 'codex', 'idle', ${DateTime.formatIso(now)},
874 '{"obsolete":true}')`;
875 yield* orchestrator.dispatch({
876 type: "message.dispatch",
877 commandId: CommandId.make("dispatch-with-old-history"),
878 threadId,
879 messageId: MessageId.make("fresh-input"),
880 text: "Continue",
881 attachments: [],
882 dispatchMode: { type: "defer_start" },
883 createdBy: "user",
884 creationSource: "web",
885 });
886 const fresh = yield* projections.getThreadRecords(threadId, ["turnItems"], {
887 turnItemTypes: ["user_message"],
888 });
889 assert.isAbove(fresh.turnItems.at(-1)!.ordinal, 900);
890 // Deletion reads transfer history separately to find side chats.
891 yield* sql`DELETE FROM orchestration_v2_projection_context_transfers
892 WHERE target_thread_id = ${threadId}`;
893 // Archiving reads every task of the thread to stop the ones still running.
894 yield* sql`DELETE FROM orchestration_v2_projection_subagents
895 WHERE subagent_id = 'obsolete-task'`;
896 yield* sql`DELETE FROM orchestration_v2_projection_nodes WHERE node_id = 'obsolete-task'`;
897 yield* sql`DELETE FROM orchestration_v2_projection_provider_threads
898 WHERE provider_thread_id = 'obsolete-task-thread'`;
899 yield* orchestrator.dispatch({
900 type: "thread.archive",
901 commandId: CommandId.make("archive-with-old-history"),
902 threadId,
903 });
904 yield* orchestrator.dispatch({
905 type: "thread.delete",
906 commandId: CommandId.make("delete-with-old-history"),
907 threadId,
908 });
909 assert.isNotNull((yield* projections.getThread(threadId)).deletedAt);
910 }).pipe(Effect.provide(layerTest)),
911 );
912
913 it.effect("keeps the store's read failure as the cause when a command projection fails", () =>
914 Effect.gen(function* () {
915 const orchestrator = yield* Orchestrator.OrchestratorV2;
916 const sql = yield* SqlClient.SqlClient;
917 const now = DateTime.formatIso(yield* DateTime.now);
918 const unreadableMessage = ThreadId.make("thread:control-unreadable-message");
919 const unreadableRequest = ThreadId.make("thread:control-unreadable-request");
920 for (const threadId of [unreadableMessage, unreadableRequest]) {
921 yield* orchestrator.dispatch({
922 type: "thread.create",
923 commandId: CommandId.make(`create:${threadId}`),
924 threadId,
925 projectId: ProjectId.make("project:control-unreadable"),
926 title: "Unreadable",
927 modelSelection,
928 runtimeMode: "full-access",
929 interactionMode: "default",
930 branch: null,
931 worktreePath: null,
932 createdBy: "user",
933 creationSource: "web",
934 });
935 }
936 yield* sql`INSERT INTO orchestration_v2_projection_messages
937 (message_id, thread_id, run_id, node_id, role, streaming, created_at, updated_at, payload_json)
938 VALUES ('obsolete-user', ${unreadableMessage}, NULL, NULL, 'user', 0, ${now}, ${now}, '{"obsolete":true}')`;
939 yield* sql`INSERT INTO orchestration_v2_projection_runtime_requests
940 (runtime_request_id, thread_id, node_id, kind, status, created_at, payload_json)
941 VALUES ('request:unreadable', ${unreadableRequest}, 'node:unreadable', 'user-input', 'pending', ${now}, '{}')`;
942 for (const command of [
943 {
944 type: "queue.resume",
945 commandId: CommandId.make("resume-unreadable"),
946 threadId: unreadableMessage,
947 },
948 {
949 type: "thread.user-input.dismiss",
950 commandId: CommandId.make("dismiss-unreadable"),
951 threadId: unreadableRequest,
952 requestId: RuntimeRequestId.make("request:unreadable"),
953 },
954 ] as const) {
955 const failure = yield* orchestrator.dispatch(command).pipe(Effect.flip);
956 assert.instanceOf(failure, Orchestrator.OrchestratorProjectionError);
957 assert.instanceOf(failure.cause, ProjectionStore.ProjectionStoreReadError);
958 }
959 }).pipe(Effect.provide(layerTest)),
960 );
961
962 it.effect("implements a proposed plan that the command projection leaves out", () =>
963 Effect.gen(function* () {
964 const orchestrator = yield* Orchestrator.OrchestratorV2;
965 const projections = yield* ProjectionStore.ProjectionStoreV2;
966 const threadId = ThreadId.make("thread:implement-plan");
967 const planId = PlanId.make("plan:implement-plan");
968 const now = yield* DateTime.now;
969 yield* orchestrator.dispatch({
970 type: "thread.create",
971 commandId: CommandId.make("create-implement-plan"),
972 threadId,
973 projectId: ProjectId.make("project:implement-plan"),
974 title: "Plan",
975 modelSelection,
976 runtimeMode: "full-access",
977 interactionMode: "plan",
978 branch: null,
979 worktreePath: null,
980 createdBy: "user",
981 creationSource: "web",
982 });
983 yield* projections.apply({
984 id: EventId.make("plan:implement-plan"),
985 type: "plan.updated",
986 threadId,
987 occurredAt: now,
988 payload: {
989 id: planId,
990 threadId,
991 runId: null,
992 nodeId: NodeId.make("node:implement-plan"),
993 kind: "proposed_plan",
994 status: "active",
995 markdown: "# Plan\n\n1. Do the thing.",
996 },
997 });
998
999 yield* orchestrator.dispatch({
1000 type: "message.dispatch",
1001 commandId: CommandId.make("implement-plan"),
1002 threadId,
1003 messageId: MessageId.make("implement-plan-input"),
1004 text: "Implement the plan.",
1005 attachments: [],
1006 sourcePlanRef: { threadId, planId },
1007 dispatchMode: { type: "defer_start" },
1008 createdBy: "user",
1009 creationSource: "web",
1010 });
1011
1012 assert.equal((yield* projections.getPlan(threadId, planId))?.status, "completed");
1013 }).pipe(Effect.provide(layerTest)),
1014 );
1015
1016 it.effect("settles only the stopped run's background work, once", () =>
1017 Effect.gen(function* () {
1018 const orchestrator = yield* Orchestrator.OrchestratorV2;
1019 const projections = yield* ProjectionStore.ProjectionStoreV2;
1020 const threadId = ThreadId.make("thread:settle-binding");
1021 const providerThreadId = ProviderThreadId.make("provider-thread:settle-binding");
1022 const now = yield* DateTime.now;
1023 yield* orchestrator.dispatch({
1024 type: "thread.create",
1025 commandId: CommandId.make("create-settle-binding"),
1026 threadId,
1027 projectId: ProjectId.make("project:settle-binding"),
1028 title: "Settle binding",
1029 modelSelection,
1030 runtimeMode: "full-access",
1031 interactionMode: "default",
1032 branch: null,
1033 worktreePath: null,
1034 createdBy: "user",
1035 creationSource: "web",
1036 });
1037 yield* projections.apply({
1038 id: EventId.make("settle-binding:provider-thread"),
1039 type: "provider-thread.updated",
1040 threadId,
1041 occurredAt: now,
1042 payload: {
1043 id: providerThreadId,
1044 driver: adapter.driver,
1045 providerInstanceId: instanceId,
1046 providerSessionId: null,
1047 appThreadId: threadId,
1048 ownerNodeId: null,
1049 nativeThreadRef: null,
1050 nativeConversationHeadRef: null,
1051 status: "idle",
1052 firstRunOrdinal: 1,
1053 lastRunOrdinal: 2,
1054 handoffIds: [],
1055 forkedFrom: null,
1056 createdAt: now,
1057 updatedAt: now,
1058 },
1059 });
1060 const commandItem = (ordinal: number) => TurnItemId.make(`turn-item:settle-binding:${ordinal}`);
1061 for (const ordinal of [1, 2]) {
1062 const runId = RunId.make(`run:settle-binding:${ordinal}`);
1063 const attemptId = RunAttemptId.make(`attempt:settle-binding:${ordinal}`);
1064 const nodeId = NodeId.make(`node:settle-binding:${ordinal}`);
1065 const providerTurnId = ProviderTurnId.make(`provider-turn:settle-binding:${ordinal}`);
1066 yield* projections.apply({
1067 id: EventId.make(`settle-binding:run:${ordinal}`),
1068 type: "run.created",
1069 threadId,
1070 occurredAt: now,
1071 payload: {
1072 id: runId,
1073 threadId,
1074 ordinal,
1075 providerInstanceId: instanceId,
1076 modelSelection,
1077 providerThreadId,
1078 userMessageId: MessageId.make(`message:settle-binding:${ordinal}`),
1079 rootNodeId: nodeId,
1080 activeAttemptId: attemptId,
1081 status: "completed",
1082 requestedAt: now,
1083 startedAt: now,
1084 completedAt: now,
1085 checkpointId: null,
1086 contextHandoffId: null,
1087 },
1088 });
1089 yield* projections.apply({
1090 id: EventId.make(`settle-binding:attempt:${ordinal}`),
1091 type: "run-attempt.created",
1092 threadId,
1093 occurredAt: now,
1094 payload: {
1095 id: attemptId,
1096 runId,
1097 attemptOrdinal: 1,
1098 rootNodeId: nodeId,
1099 providerInstanceId: instanceId,
1100 providerThreadId,
1101 providerTurnId,
1102 reason: "initial",
1103 status: "completed",
1104 startedAt: now,
1105 completedAt: now,
1106 },
1107 });
1108 yield* projections.apply({
1109 id: EventId.make(`settle-binding:turn:${ordinal}`),
1110 type: "provider-turn.updated",
1111 threadId,
1112 occurredAt: now,
1113 payload: {
1114 id: providerTurnId,
1115 providerThreadId,
1116 nodeId,
1117 runAttemptId: attemptId,
1118 nativeTurnRef: null,
1119 ordinal,
1120 status: "completed",
1121 startedAt: now,
1122 completedAt: now,
1123 },
1124 });
1125 yield* projections.apply({
1126 id: EventId.make(`settle-binding:item:${ordinal}`),
1127 type: "turn-item.updated",
1128 threadId,
1129 runId,
1130 occurredAt: now,
1131 payload: {
1132 id: commandItem(ordinal),
1133 threadId,
1134 runId,
1135 nodeId,
1136 providerThreadId,
1137 providerTurnId,
1138 nativeItemRef: null,
1139 parentItemId: null,
1140 ordinal: ordinal * 10,
1141 status: "running",
1142 title: `Background command ${ordinal}`,
1143 startedAt: now,
1144 completedAt: null,
1145 updatedAt: now,
1146 type: "command_execution",
1147 input: `sleep ${ordinal}`,
1148 },
1149 });
1150 }
1151 const itemStatuses = Effect.map(projections.getThreadProjection(threadId), (projection) =>
1152 projection.turnItems
1153 .flatMap((item) => (item.type === "command_execution" ? [`${item.id}:${item.status}`] : []))
1154 .toSorted(),
1155 );
1156 const settle = {
1157 type: "thread.background-work.settle",
1158 commandId: CommandId.make("stop-run-1:background-work-settled"),
1159 threadId,
1160 providerThreadId,
1161 providerTurnId: ProviderTurnId.make("provider-turn:settle-binding:1"),
1162 } as const;
1163 yield* orchestrator.dispatch(settle);
1164 assert.deepEqual(yield* itemStatuses, [
1165 `${commandItem(1)}:interrupted`,
1166 `${commandItem(2)}:running`,
1167 ]);
1168
1169 const emptySettle = {
1170 ...settle,
1171 commandId: CommandId.make("stop-run-1-again:background-work-settled"),
1172 };
1173 const first = yield* orchestrator.dispatch(emptySettle);
1174 assert.lengthOf(first.storedEvents, 0);
1175 yield* projections.apply({
1176 id: EventId.make("settle-binding:item:late"),
1177 type: "turn-item.updated",
1178 threadId,
1179 runId: RunId.make("run:settle-binding:1"),
1180 occurredAt: now,
1181 payload: {
1182 id: commandItem(3),
1183 threadId,
1184 runId: RunId.make("run:settle-binding:1"),
1185 nodeId: NodeId.make("node:settle-binding:1"),
1186 providerThreadId,
1187 providerTurnId: ProviderTurnId.make("provider-turn:settle-binding:1"),
1188 nativeItemRef: null,
1189 parentItemId: null,
1190 ordinal: 30,
1191 status: "running",
1192 title: "Late background command",
1193 startedAt: now,
1194 completedAt: null,
1195 updatedAt: now,
1196 type: "command_execution",
1197 input: "sleep 3",
1198 },
1199 });
1200 const replayed = yield* orchestrator.dispatch(emptySettle);
1201 assert.lengthOf(replayed.storedEvents, 0);
1202 assert.deepEqual(yield* itemStatuses, [
1203 `${commandItem(1)}:interrupted`,
1204 `${commandItem(2)}:running`,
1205 `${commandItem(3)}:running`,
1206 ]);
1207 }).pipe(Effect.provide(layerTest)),
1208 );
1209
1210 it.effect("keeps delegated child pull-request links independent of the parent", () =>
1211 Effect.gen(function* () {
1212 const orchestrator = yield* Orchestrator.OrchestratorV2;
1213 const projections = yield* ProjectionStore.ProjectionStoreV2;
1214 const parentThreadId = ThreadId.make("thread:parent-pr");
1215 const projectId = ProjectId.make("project:parent-pr");
1216 const parentPullRequest = {
1217 projectId,
1218 repository: "pingdotgg/t3code",
1219 number: 123,
1220 url: "https://github.com/pingdotgg/t3code/pull/123",
1221 };
1222 yield* orchestrator.dispatch({
1223 type: "thread.create",
1224 commandId: CommandId.make("create-parent-pr"),
1225 threadId: parentThreadId,
1226 projectId,
1227 title: "Parent with a linked PR",
1228 modelSelection,
1229 runtimeMode: "full-access",
1230 interactionMode: "default",
1231 branch: "feature/parent-pr",
1232 worktreePath: "/repo-worktree",
1233 createdBy: "user",
1234 creationSource: "web",
1235 });
1236 yield* orchestrator.dispatch({
1237 type: "thread.metadata.update",
1238 commandId: CommandId.make("link-parent-pr"),
1239 threadId: parentThreadId,
1240 linkedPullRequest: parentPullRequest,
1241 });
1242 yield* orchestrator.dispatch({
1243 type: "message.dispatch",
1244 commandId: CommandId.make("start-parent-pr"),
1245 threadId: parentThreadId,
1246 messageId: MessageId.make("message:parent-pr"),
1247 text: "Delegate a review",
1248 attachments: [],
1249 dispatchMode: { type: "start_immediately" },
1250 createdBy: "user",
1251 creationSource: "web",
1252 });
1253 const parent = yield* projections.getThreadProjection(parentThreadId);
1254 const parentRun = parent.runs[0]!;
1255 const failedSpans: Array<string> = [];
1256 const tracer = Tracer.make({
1257 span(options) {
1258 const span = new Tracer.NativeSpan(options);
1259 const end = span.end.bind(span);
1260 span.end = (time, exit) => {
1261 end(time, exit);
1262 if (Exit.isFailure(exit)) failedSpans.push(span.name);
1263 };
1264 return span;
1265 },
1266 });
1267 yield* orchestrator
1268 .dispatch({
1269 type: "delegated_task.request",
1270 commandId: CommandId.make("delegate-parent-pr"),
1271 parentThreadId,
1272 parentRunId: parentRun.id,
1273 parentNodeId: parentRun.rootNodeId!,
1274 task: "Review the changes",
1275 modelSelection,
1276 runtimeMode: "full-access",
1277 interactionMode: "default",
1278 createdBy: "agent",
1279 creationSource: "mcp",
1280 })
1281 .pipe(Effect.withTracer(tracer));
1282 // The child thread is created by this command; a failed span here reads as an error in traces.
1283 assert.deepEqual(failedSpans, []);
1284 const updatedParent = yield* projections.getThreadProjection(parentThreadId);
1285 assert.deepEqual(updatedParent.subagents[0]!.modelSelection, modelSelection);
1286 const childThreadId = updatedParent.subagents[0]!.childThreadId!;
1287 const child = yield* projections.getThreadProjection(childThreadId);
1288 assert.isNull(child.thread.linkedPullRequest);
1289 assert.deepEqual(child.thread.pullRequests, []);
1290 assert.equal(child.thread.branch, parent.thread.branch);
1291 assert.equal(child.thread.worktreePath, parent.thread.worktreePath);
1292 assert.equal(child.thread.lineage.parentThreadId, parentThreadId);
1293
1294 const childPullRequest = {
1295 ...parentPullRequest,
1296 number: 456,
1297 url: "https://github.com/pingdotgg/t3code/pull/456",
1298 };
1299 yield* orchestrator.dispatch({
1300 type: "thread.metadata.update",
1301 commandId: CommandId.make("link-child-pr"),
1302 threadId: childThreadId,
1303 linkedPullRequest: childPullRequest,
1304 });
1305 const linkedChild = yield* projections.getThreadProjection(childThreadId);
1306 assert.deepEqual(linkedChild.thread.linkedPullRequest, childPullRequest);
1307 assert.deepEqual(
1308 linkedChild.thread.pullRequests?.map((link) => link.number),
1309 [456],
1310 );
1311 const parentAfterChildLink = yield* projections.getThreadProjection(parentThreadId);
1312 assert.deepEqual(parentAfterChildLink.thread.linkedPullRequest, parentPullRequest);
1313 assert.deepEqual(parentAfterChildLink.thread.pullRequests, parent.thread.pullRequests);
1314 }).pipe(Effect.provide(layerTest)),
1315 );
1316
1317 it.effect("wakes the parent once when its delegated task asks a question", () =>
1318 Effect.gen(function* () {
1319 const orchestrator = yield* Orchestrator.OrchestratorV2;
1320 const projections = yield* ProjectionStore.ProjectionStoreV2;
1321 const reactor = yield* DelegatedTaskQuestionReactor.DelegatedTaskQuestionReactor;
1322 const parentThreadId = ThreadId.make("thread:question-parent");
1323 yield* orchestrator.dispatch({
1324 type: "thread.create",
1325 commandId: CommandId.make("create-question-parent"),
1326 threadId: parentThreadId,
1327 projectId: ProjectId.make("project:question"),
1328 title: "Parent",
1329 modelSelection,
1330 runtimeMode: "full-access",
1331 interactionMode: "default",
1332 branch: null,
1333 worktreePath: null,
1334 createdBy: "user",
1335 creationSource: "web",
1336 });
1337 yield* orchestrator.dispatch({
1338 type: "message.dispatch",
1339 commandId: CommandId.make("start-question-parent"),
1340 threadId: parentThreadId,
1341 messageId: MessageId.make("message:question-parent"),
1342 text: "Delegate the research",
1343 attachments: [],
1344 dispatchMode: { type: "start_immediately" },
1345 createdBy: "user",
1346 creationSource: "web",
1347 });
1348 const parentRun = (yield* projections.getThreadProjection(parentThreadId)).runs[0]!;
1349 yield* orchestrator.dispatch({
1350 type: "delegated_task.request",
1351 commandId: CommandId.make("delegate-question-task"),
1352 parentThreadId,
1353 parentRunId: parentRun.id,
1354 parentNodeId: parentRun.rootNodeId!,
1355 task: "Research the sources",
1356 completionWake: "settled_only",
1357 modelSelection,
1358 runtimeMode: "full-access",
1359 interactionMode: "default",
1360 createdBy: "agent",
1361 creationSource: "mcp",
1362 });
1363 const task = (yield* projections.getThreadProjection(parentThreadId)).subagents[0]!;
1364 const childRun = (yield* projections.getThreadRecords(task.childThreadId!, ["runs"])).runs[0]!;
1365 const now = yield* DateTime.now;
1366 const requestEvent = (
1367 eventId: string,
1368 requestId: string,
1369 responseCapability: "live" | "message" = "message",
1370 kind: "user_input" | "command" = "user_input",
1371 run = childRun,
1372 ) =>
1373 ({
1374 id: EventId.make(eventId),
1375 type: "runtime-request.updated",
1376 threadId: task.childThreadId!,
1377 runId: run.id,
1378 occurredAt: now,
1379 payload: {
1380 id: RuntimeRequestId.make(requestId),
1381 nodeId: run.rootNodeId!,
1382 providerTurnId: null,
1383 nativeRequestRef: null,
1384 kind,
1385 status: "pending",
1386 responseCapability:
1387 responseCapability === "live"
1388 ? { type: "live", providerSessionId: ProviderSessionId.make("session:question") }
1389 : { type: "message" },
1390 createdAt: now,
1391 resolvedAt: null,
1392 },
1393 }) satisfies OrchestrationV2DomainEvent;
1394 const question = requestEvent("event:question", "request:question", "live");
1395 const asyncQuestion = requestEvent("event:async-question", "request:async-question");
1396 const answered = requestEvent("event:answered", "request:answered", "live");
1397 const approval = requestEvent("event:approval", "request:approval", "message", "command");
1398 const eventSink = yield* EventSink.EventSinkV2;
1399 yield* eventSink.write({
1400 events: [question, asyncQuestion, answered, approval],
1401 });
1402
1403 yield* reactor.handle(question);
1404 assert.isFalse(
1405 (yield* projections.getThreadProjection(parentThreadId)).messages.some(
1406 (message) => message.notification !== undefined,
1407 ),
1408 );
1409 yield* reactor.handle(asyncQuestion);
1410 assert.equal(
1411 (yield* projections.getThreadProjection(parentThreadId)).messages.filter(
1412 (message) => message.notification !== undefined,
1413 ).length,
1414 1,
1415 );
1416 yield* reactor.handle(asyncQuestion);
1417 yield* reactor.handle(answered);
1418 yield* eventSink.write({
1419 events: [
1420 {
1421 ...answered,
1422 id: EventId.make("event:answered-resolved"),
1423 payload: { ...answered.payload, status: "resolved", resolvedAt: now },
1424 },
1425 ],
1426 });
1427 yield* reactor.handle(answered);
1428 yield* reactor.handle(approval);
1429
1430 const wakes = (yield* projections.getThreadProjection(parentThreadId)).messages.filter(
1431 (message) => message.notification !== undefined,
1432 );
1433 assert.include(wakes[0]?.text, "asked a question and is still working");
1434 assert.deepEqual(
1435 wakes.map((message) => message.notification),
1436 [
1437 {
1438 source: {
1439 kind: "delegated_task",
1440 taskIds: [task.id],
1441 childThreadId: task.childThreadId!,
1442 },
1443 outcome: "updated",
1444 summary: 'Delegated task "Research the sources" has a question',
1445 },
1446 ],
1447 );
1448
1449 const wakeRun = (yield* projections.getThreadRecords(parentThreadId, ["runs"])).runs.find(
1450 (run) => run.userMessageId === wakes[0]!.id,
1451 )!;
1452 assert.equal(wakeRun.status, "queued");
1453 const resolved = {
1454 ...asyncQuestion,
1455 id: EventId.make("event:question-resolved"),
1456 payload: { ...asyncQuestion.payload, status: "resolved" as const, resolvedAt: now },
1457 };
1458 yield* eventSink.write({ events: [resolved] });
1459 yield* reactor.handle(resolved);
1460 assert.equal(
1461 (yield* projections.getThreadRecords(parentThreadId, ["runs"])).runs.find(
1462 (run) => run.id === wakeRun.id,
1463 )?.status,
1464 "cancelled",
1465 );
1466 const parentCompleted = {
1467 id: EventId.make("event:question-parent-completed"),
1468 type: "run.updated",
1469 threadId: parentThreadId,
1470 runId: parentRun.id,
1471 occurredAt: now,
1472 payload: { ...parentRun, status: "completed", completedAt: now },
1473 } satisfies OrchestrationV2DomainEvent;
1474 for (const status of [
1475 "preparing",
1476 "starting",
1477 "running",
1478 "queued",
1479 "cancelled",
1480 "rolled_back",
1481 "interrupted",
1482 ] as const) {
1483 yield* reactor.handle({ ...parentCompleted, payload: { ...parentRun, status } });
1484 }
1485 assert.equal(
1486 (yield* projections.getThreadRecords(parentThreadId, ["messages"])).messages.length,
1487 2,
1488 );
1489 yield* eventSink.write({ events: [parentCompleted] });
1490 for (const state of ["disposed", "pending"] as const) {
1491 yield* eventSink.write({
1492 events: [
1493 {
1494 id: EventId.make(`event:question-task-${state}`),
1495 type: "subagent.updated",
1496 threadId: parentThreadId,
1497 occurredAt: now,
1498 payload: { ...task, completionDelivery: { state, observedByRunId: null } },
1499 },
1500 ],
1501 });
1502 yield* reactor.handle(parentCompleted);
1503 if (state === "disposed") {
1504 assert.equal(
1505 (yield* projections.getThreadRecords(parentThreadId, ["messages"])).messages.length,
1506 2,
1507 );
1508 }
1509 }
1510 yield* reactor.handle(parentCompleted);
1511 const caughtUp = (yield* projections.getThreadProjection(parentThreadId)).messages.filter(
1512 (message) => message.notification !== undefined,
1513 );
1514 assert.equal(caughtUp.length, 2);
1515 assert.equal(
1516 caughtUp.filter((message) => message.text.includes(`requestId ${question.payload.id}.`))
1517 .length,
1518 1,
1519 );
1520 assert.include(caughtUp[1]?.text, "is blocked on a question");
1521 assert.isFalse(
1522 caughtUp.some((message) => message.text.includes(`requestId ${answered.payload.id}.`)),
1523 );
1524 yield* reactor.handle(question);
1525 assert.equal(
1526 (yield* projections.getThreadProjection(parentThreadId)).messages.filter(
1527 (message) => message.notification !== undefined,
1528 ).length,
1529 2,
1530 );
1531 const terminalQuestion = requestEvent(
1532 "event:terminal-question",
1533 "request:terminal-question",
1534 "live",
1535 );
1536 yield* eventSink.write({ events: [terminalQuestion] });
1537 yield* reactor.handle(terminalQuestion);
1538 assert.equal(
1539 (yield* projections.getThreadProjection(parentThreadId)).messages.filter(
1540 (message) => message.notification !== undefined,
1541 ).length,
1542 3,
1543 );
1544 yield* eventSink.write({
1545 events: [
1546 {
1547 id: EventId.make("event:question-parent-running-again"),
1548 type: "run.updated",
1549 threadId: parentThreadId,
1550 runId: parentRun.id,
1551 occurredAt: now,
1552 payload: { ...parentRun, status: "running", completedAt: null },
1553 },
1554 ],
1555 });
1556 yield* orchestrator.dispatch({
1557 type: "delegated_task.wake-policy",
1558 commandId: CommandId.make("terminal-question-wait-ended"),
1559 parentThreadId,
1560 taskId: task.id,
1561 completionWake: "always",
1562 });
1563 const upgradedQuestion = requestEvent(
1564 "event:upgraded-question",
1565 "request:upgraded-question",
1566 "live",
1567 );
1568 yield* eventSink.write({ events: [upgradedQuestion] });
1569 yield* reactor.handle(upgradedQuestion);
1570 assert.equal(
1571 (yield* projections.getThreadProjection(parentThreadId)).messages.filter(
1572 (message) => message.notification !== undefined,
1573 ).length,
1574 4,
1575 );
1576 const waitDelivered = requestEvent("event:wait-delivered", "request:wait-delivered", "live");
1577 const waitDeliveredParentCompleted = {
1578 ...parentCompleted,
1579 id: EventId.make("event:wait-delivered-parent-completed"),
1580 };
1581 yield* eventSink.write({ events: [waitDelivered, waitDeliveredParentCompleted] });
1582 yield* reactor.handle(waitDeliveredParentCompleted);
1583 assert.isFalse(
1584 (yield* projections.getThreadProjection(parentThreadId)).messages.some((message) =>
1585 message.text.includes(`requestId ${waitDelivered.payload.id}.`),
1586 ),
1587 );
1588 yield* eventSink.write({
1589 commandId: CommandId.make("command:runtime-reconcile:question-followup"),
1590 events: [
1591 {
1592 id: EventId.make("event:question-child-completed"),
1593 type: "run.updated",
1594 threadId: task.childThreadId!,
1595 occurredAt: now,
1596 payload: { ...childRun, status: "completed", completedAt: now },
1597 },
1598 {
1599 id: EventId.make("event:question-task-completed"),
1600 type: "subagent.updated",
1601 threadId: parentThreadId,
1602 occurredAt: now,
1603 payload: {
1604 ...task,
1605 status: "completed",
1606 result: "Sources researched.",
1607 completedAt: now,
1608 },
1609 },
1610 ],
1611 });
1612 yield* orchestrator.dispatch({
1613 type: "message.dispatch",
1614 commandId: CommandId.make("question-followup"),
1615 threadId: task.childThreadId!,
1616 messageId: MessageId.make("message:question-followup"),
1617 text: "Check the sources again",
1618 attachments: [],
1619 dispatchMode: { type: "start_immediately" },
1620 createdBy: "user",
1621 creationSource: "web",
1622 });
1623 const followup = (yield* projections.getThreadRecords(task.childThreadId!, ["runs"])).runs.find(
1624 (run) => run.userMessageId === "message:question-followup",
1625 )!;
1626 assert.isDefined(followup.delegatedTaskId);
1627 assert.notEqual(followup.delegatedTaskId, task.id);
1628 const followupQuestion = requestEvent(
1629 "event:followup-question",
1630 "request:followup-question",
1631 "live",
1632 "user_input",
1633 followup,
1634 );
1635 yield* eventSink.write({ events: [followupQuestion] });
1636 yield* reactor.handle(followupQuestion);
1637 const followupWake = (yield* projections.getThreadRecords(parentThreadId, [
1638 "messages",
1639 ])).messages.find((message) =>
1640 message.text.includes(`requestId ${followupQuestion.payload.id}`),
1641 )!;
1642 assert.deepEqual(followupWake.notification?.source, {
1643 kind: "delegated_task",
1644 taskIds: [followup.delegatedTaskId],
1645 childThreadId: task.childThreadId!,
1646 });
1647 }).pipe(Effect.provide(DelegatedTaskQuestionReactor.layer.pipe(Layer.provideMerge(layerTest)))),
1648 );
1649
1650 it.effect.each([false, true])(
1651 "drops a stale question wake (missed terminal event: %s)",
1652 (missedEvent) =>
1653 Effect.gen(function* () {
1654 const orchestrator = yield* Orchestrator.OrchestratorV2;
1655 const projections = yield* ProjectionStore.ProjectionStoreV2;
1656 const reactor = yield* DelegatedTaskQuestionReactor.DelegatedTaskQuestionReactor;
1657 const eventSink = yield* EventSink.EventSinkV2;
1658 const parentThreadId = ThreadId.make("thread:stale-question-parent");
1659 yield* orchestrator.dispatch({
1660 type: "thread.create",
1661 commandId: CommandId.make("create-stale-question-parent"),
1662 threadId: parentThreadId,
1663 projectId: ProjectId.make("project:stale-question"),
1664 title: "Parent",
1665 modelSelection,
1666 runtimeMode: "full-access",
1667 interactionMode: "default",
1668 branch: null,
1669 worktreePath: null,
1670 createdBy: "user",
1671 creationSource: "web",
1672 });
1673 yield* orchestrator.dispatch({
1674 type: "message.dispatch",
1675 commandId: CommandId.make("start-stale-question-parent"),
1676 threadId: parentThreadId,
1677 messageId: MessageId.make("message:stale-question-parent"),
1678 text: "Delegate the proof",
1679 attachments: [],
1680 dispatchMode: { type: "start_immediately" },
1681 createdBy: "user",
1682 creationSource: "web",
1683 });
1684 const parentRun = (yield* projections.getThreadProjection(parentThreadId)).runs[0]!;
1685 yield* orchestrator.dispatch({
1686 type: "delegated_task.request",
1687 commandId: CommandId.make("delegate-stale-question-task"),
1688 parentThreadId,
1689 parentRunId: parentRun.id,
1690 parentNodeId: parentRun.rootNodeId!,
1691 task: "Capture the proof",
1692 completionWake: "settled_only",
1693 modelSelection,
1694 runtimeMode: "full-access",
1695 interactionMode: "default",
1696 createdBy: "agent",
1697 creationSource: "mcp",
1698 });
1699 const task = (yield* projections.getThreadProjection(parentThreadId)).subagents[0]!;
1700 const childRun = (yield* projections.getThreadRecords(task.childThreadId!, ["runs"]))
1701 .runs[0]!;
1702 const now = yield* DateTime.now;
1703 const question = {
1704 id: EventId.make("event:stale-question"),
1705 type: "runtime-request.updated",
1706 threadId: task.childThreadId!,
1707 runId: childRun.id,
1708 occurredAt: now,
1709 payload: {
1710 id: RuntimeRequestId.make("request:stale-question"),
1711 nodeId: childRun.rootNodeId!,
1712 providerTurnId: null,
1713 nativeRequestRef: null,
1714 kind: "user_input",
1715 status: "pending",
1716 responseCapability: { type: "message" },
1717 createdAt: now,
1718 resolvedAt: null,
1719 },
1720 } satisfies OrchestrationV2DomainEvent;
1721 yield* eventSink.write({ events: [question] });
1722 yield* reactor.handle(question);
1723 const wake = (yield* projections.getThreadRecords(parentThreadId, [
1724 "messages",
1725 ])).messages.find((message) => message.notification !== undefined)!;
1726 const wakeRun = () =>
1727 projections
1728 .getThreadRecords(parentThreadId, ["runs"])
1729 .pipe(Effect.map(({ runs }) => runs.find((run) => run.userMessageId === wake.id)));
1730 assert.equal((yield* wakeRun())?.status, "queued");
1731
1732 // The child finishes without ever resolving its message-capability question.
1733 const childCompleted = {
1734 id: EventId.make("event:stale-question-child-completed"),
1735 type: "run.updated",
1736 threadId: task.childThreadId!,
1737 runId: childRun.id,
1738 occurredAt: now,
1739 payload: { ...childRun, status: "completed", completedAt: now },
1740 } satisfies OrchestrationV2DomainEvent;
1741 const taskCompleted = {
1742 id: EventId.make("event:stale-question-task-completed"),
1743 type: "subagent.updated",
1744 threadId: parentThreadId,
1745 occurredAt: now,
1746 payload: { ...task, status: "completed", result: "Proof captured.", completedAt: now },
1747 } satisfies OrchestrationV2DomainEvent;
1748 yield* eventSink.write({
1749 commandId: CommandId.make("command:runtime-reconcile:stale-question"),
1750 events: [childCompleted, taskCompleted],
1751 });
1752 yield* reactor.handle(childCompleted);
1753 if (missedEvent) {
1754 // An acknowledged result drops the task from the parent's command projection.
1755 yield* orchestrator.dispatch({
1756 type: "delegated_task.completion-delivery.acknowledge",
1757 commandId: CommandId.make("ack-stale-question-task"),
1758 parentThreadId,
1759 taskId: task.id,
1760 observedByRunId: parentRun.id,
1761 });
1762 // Persisted terminal state can predate the reactor subscription after restart.
1763 yield* projections.apply({
1764 id: EventId.make("event:stale-question-parent-finished"),
1765 type: "run.updated",
1766 threadId: parentThreadId,
1767 occurredAt: now,
1768 payload: { ...parentRun, status: "completed", completedAt: now },
1769 });
1770 yield* orchestrator.dispatch({
1771 type: "queue.resume",
1772 commandId: CommandId.make("resume-stale-question"),
1773 threadId: parentThreadId,
1774 });
1775 } else {
1776 yield* reactor.handle(taskCompleted);
1777 }
1778 assert.equal((yield* wakeRun())?.status, "cancelled");
1779 if (missedEvent) {
1780 const before = (yield* projections.getThreadRecords(parentThreadId, ["runs"])).runs.length;
1781 const delayedQuestion = {
1782 ...question,
1783 id: EventId.make("event:delayed-question"),
1784 payload: { ...question.payload, id: RuntimeRequestId.make("request:delayed-question") },
1785 };
1786 yield* eventSink.write({ events: [delayedQuestion] });
1787 yield* reactor.handle(delayedQuestion);
1788 // A stale delivery is accepted without starting work, not rejected.
1789 yield* orchestrator.dispatch({
1790 type: "message.dispatch",
1791 commandId: CommandId.make("stale-question-redelivery"),
1792 threadId: parentThreadId,
1793 messageId: MessageId.make("message:stale-question-redelivery"),
1794 text: "Answer the delegated question",
1795 notification: {
1796 source: {
1797 kind: "delegated_task",
1798 taskIds: [task.id],
1799 childThreadId: task.childThreadId!,
1800 },
1801 outcome: "updated",
1802 summary: "Delegated task has a question",
1803 },
1804 attachments: [],
1805 dispatchMode: { type: "queue_after_active" },
1806 createdBy: "agent",
1807 creationSource: "server",
1808 });
1809 assert.equal(
1810 (yield* projections.getThreadRecords(parentThreadId, ["runs"])).runs.length,
1811 before,
1812 );
1813 return;
1814 }
1815
1816 yield* orchestrator.dispatch({
1817 type: "message.dispatch",
1818 commandId: CommandId.make("stale-question-followup"),
1819 threadId: task.childThreadId!,
1820 messageId: MessageId.make("message:stale-question-followup"),
1821 text: "Capture another proof",
1822 attachments: [],
1823 dispatchMode: { type: "start_immediately" },
1824 createdBy: "user",
1825 creationSource: "web",
1826 });
1827 const followup = (yield* projections.getThreadRecords(task.childThreadId!, [
1828 "runs",
1829 ])).runs.find((run) => run.userMessageId === "message:stale-question-followup")!;
1830 const followupQuestion = {
1831 ...question,
1832 id: EventId.make("event:stale-question-followup"),
1833 runId: followup.id,
1834 payload: {
1835 ...question.payload,
1836 id: RuntimeRequestId.make("request:stale-question-followup"),
1837 nodeId: followup.rootNodeId!,
1838 },
1839 };
1840 yield* eventSink.write({ events: [followupQuestion] });
1841 yield* reactor.handle(followupQuestion);
1842 const followupWake = (yield* projections.getThreadRecords(parentThreadId, [
1843 "runs",
1844 ])).runs.find(
1845 (run) =>
1846 run.userMessageId ===
1847 `message:delegated-task-question:${parentThreadId}:${followupQuestion.payload.id}`,
1848 )!;
1849 assert.equal(followupWake.status, "queued");
1850 yield* reactor.handle(taskCompleted);
1851 assert.equal(
1852 (yield* projections.getThreadRecords(parentThreadId, ["runs"])).runs.find(
1853 (run) => run.id === followupWake.id,
1854 )?.status,
1855 "queued",
1856 );
1857 yield* projections.apply({
1858 id: EventId.make("event:followup-question-parent-finished"),
1859 type: "run.updated",
1860 threadId: parentThreadId,
1861 occurredAt: now,
1862 payload: { ...parentRun, status: "completed", completedAt: now },
1863 });
1864 yield* orchestrator.dispatch({
1865 type: "queue.resume",
1866 commandId: CommandId.make("resume-followup-question"),
1867 threadId: parentThreadId,
1868 });
1869 assert.equal(
1870 (yield* projections.getThreadRecords(parentThreadId, ["runs"])).runs.find(
1871 (run) => run.id === followupWake.id,
1872 )?.status,
1873 "starting",
1874 );
1875 }).pipe(Effect.provide(DelegatedTaskQuestionReactor.layer.pipe(Layer.provideMerge(layerTest)))),
1876 );
1877
1878 it.effect.each([false, true])("rejects thread.create for an existing id (deleted: %s)", (deleted) =>
1879 Effect.gen(function* () {
1880 const orchestrator = yield* Orchestrator.OrchestratorV2;
1881 const projections = yield* ProjectionStore.ProjectionStoreV2;
1882 const threadId = ThreadId.make(`thread:duplicate-create:${deleted}`);
1883 const create = {
1884 type: "thread.create",
1885 commandId: CommandId.make(`create:${threadId}`),
1886 threadId,
1887 projectId: ProjectId.make("project:duplicate-create"),
1888 title: "Original",
1889 modelSelection,
1890 runtimeMode: "full-access",
1891 interactionMode: "default",
1892 branch: null,
1893 worktreePath: null,
1894 createdBy: "user",
1895 creationSource: "web",
1896 importedNativeThread: {
1897 ref: { driver: adapter.driver, nativeId: "native-original", strength: "strong" },
1898 },
1899 } as const;
1900 const failedSpans: Array<string> = [];
1901 const tracer = Tracer.make({
1902 span(options) {
1903 const span = new Tracer.NativeSpan(options);
1904 const end = span.end.bind(span);
1905 span.end = (time, exit) => {
1906 end(time, exit);
1907 if (Exit.isFailure(exit)) failedSpans.push(span.name);
1908 };
1909 return span;
1910 },
1911 });
1912 yield* orchestrator.dispatch(create).pipe(Effect.withTracer(tracer));
1913 // The id check expects a miss; a failed span there reads as an error in Tempo.
1914 assert.deepEqual(failedSpans, []);
1915 if (deleted) {
1916 yield* orchestrator.dispatch({
1917 type: "thread.delete",
1918 commandId: CommandId.make(`delete:${threadId}`),
1919 threadId,
1920 });
1921 }
1922 const before = yield* projections.getThread(threadId);
1923 assert.isNotNull(before.activeProviderThreadId);
1924 assert.equal(before.deletedAt !== null, deleted);
1925 const sequence = yield* orchestrator.getThreadEventSequence(threadId);
1926 yield* TestClock.adjust("1 second");
1927 const duplicate = {
1928 ...create,
1929 commandId: CommandId.make(`duplicate:${threadId}`),
1930 title: "Replacement",
1931 importedNativeThread: undefined,
1932 };
1933 const outcome = yield* orchestrator.dispatch(duplicate).pipe(Effect.result);
1934 assert.ok(Result.isFailure(outcome), "duplicate create must be rejected");
1935 const refused = outcome.failure;
1936 assert.equal(refused._tag, "OrchestratorThreadAlreadyExistsError");
1937 assert.equal(refused.message, `Thread ${threadId} already exists. Use a new thread id.`);
1938 assert.deepEqual(yield* projections.getThread(threadId), before);
1939 assert.equal(yield* orchestrator.getThreadEventSequence(threadId), sequence);
1940 const retried = yield* orchestrator.dispatch(duplicate).pipe(Effect.flip);
1941 assert.equal(retried._tag, "OrchestratorCommandPreviouslyRejectedError");
1942 assert.include(retried.message, refused.message);
1943 }).pipe(Effect.provide(layerTest)),
1944 );
1945
1946 it.effect("starts, finishes and follows up a task without reading the parent's history", () =>
1947 Effect.gen(function* () {
1948 const orchestrator = yield* Orchestrator.OrchestratorV2;
1949 const projections = yield* ProjectionStore.ProjectionStoreV2;
1950 const eventSink = yield* EventSink.EventSinkV2;
1951 const sql = yield* SqlClient.SqlClient;
1952 const parentThreadId = ThreadId.make("thread:long-parent");
1953 const now = yield* DateTime.now;
1954 yield* orchestrator.dispatch({
1955 type: "thread.create",
1956 commandId: CommandId.make("create-long-parent"),
1957 threadId: parentThreadId,
1958 projectId: ProjectId.make("project:long-parent"),
1959 title: "Parent",
1960 modelSelection,
1961 runtimeMode: "full-access",
1962 interactionMode: "default",
1963 branch: null,
1964 worktreePath: null,
1965 createdBy: "user",
1966 creationSource: "web",
1967 });
1968 yield* orchestrator.dispatch({
1969 type: "message.dispatch",
1970 commandId: CommandId.make("start-long-parent"),
1971 threadId: parentThreadId,
1972 messageId: MessageId.make("message:long-parent"),
1973 text: "Delegate the work",
1974 attachments: [],
1975 dispatchMode: { type: "start_immediately" },
1976 createdBy: "user",
1977 creationSource: "web",
1978 });
1979 const parent = yield* projections.getThreadRecords(parentThreadId, [
1980 "runs",
1981 "attempts",
1982 "providerThreads",
1983 ]);
1984 const parentRun = parent.runs[0]!;
1985 const providerTurnId = ProviderTurnId.make("provider-turn:long-parent");
1986 const nativeRef = (nativeId: string) => ({
1987 driver: adapter.driver,
1988 nativeId,
1989 strength: "strong" as const,
1990 });
1991 for (const event of [
1992 {
1993 type: "provider-thread.updated",
1994 payload: { ...parent.providerThreads[0]!, nativeThreadRef: nativeRef("native:parent") },
1995 },
1996 // A steered run: its attempt points at a provider turn that another attempt started.
1997 {
1998 type: "provider-turn.updated",
1999 payload: {
2000 id: providerTurnId,
2001 providerThreadId: parentRun.providerThreadId!,
2002 nodeId: parentRun.rootNodeId!,
2003 runAttemptId: null,
2004 nativeTurnRef: nativeRef("native:parent:turn"),
2005 ordinal: 1,
2006 status: "running",
2007 startedAt: now,
2008 completedAt: null,
2009 },
2010 },
2011 { type: "run-attempt.updated", payload: { ...parent.attempts[0]!, providerTurnId } },
2012 ] as const) {
2013 yield* projections.apply({
2014 ...event,
2015 id: EventId.make(`event:long-parent:${event.type}`),
2016 threadId: parentThreadId,
2017 runId: parentRun.id,
2018 occurredAt: now,
2019 });
2020 }
2021
2022 // Keep the statements of the task start, finish and follow-up, not of the recovery around them.
2023 const taskSpans = [
2024 "orchestrationV2.dispatch.delegatedTaskRequest",
2025 "Orchestrator.finalizeAppOwnedSubagent",
2026 "test.followUp",
2027 ];
2028 const statements: Array<readonly [string, ReadonlyArray<unknown>]> = [];
2029 const recordStatements = Effect.provideService(
2030 Statement.CurrentTransformer,
2031 (statement, _make, _fiber, span) => {
2032 for (
2033 let parent = Option.some<Tracer.AnySpan>(span);
2034 Option.isSome(parent) && parent.value._tag === "Span";
2035 parent = parent.value.parent
2036 ) {
2037 if (taskSpans.includes(parent.value.name)) {
2038 statements.push(statement.compile());
2039 break;
2040 }
2041 }
2042 return Effect.succeed(statement);
2043 },
2044 );
2045 yield* orchestrator
2046 .dispatch({
2047 type: "delegated_task.request",
2048 commandId: CommandId.make("delegate-long-parent"),
2049 parentThreadId,
2050 parentRunId: parentRun.id,
2051 parentNodeId: parentRun.rootNodeId!,
2052 task: "Do the work",
2053 completionWake: "always",
2054 modelSelection,
2055 runtimeMode: "full-access",
2056 interactionMode: "default",
2057 createdBy: "agent",
2058 creationSource: "mcp",
2059 })
2060 .pipe(recordStatements);
2061 const started = yield* projections.getThreadRecords(parentThreadId, ["subagents", "turnItems"]);
2062 const task = started.subagents[0]!;
2063 const childThreadId = task.childThreadId!;
2064 assert.equal(
2065 started.turnItems.find((item) => item.type === "subagent")?.providerTurnId,
2066 providerTurnId,
2067 );
2068 const child = yield* projections.getThreadRecords(childThreadId, ["runs", "contextTransfers"]);
2069 const spawnPoint = child.contextTransfers[0]?.sourcePoint;
2070 assert.deepEqual(
2071 [spawnPoint?.providerThreadRef, spawnPoint?.providerTurnRef],
2072 [nativeRef("native:parent"), nativeRef("native:parent:turn")],
2073 );
2074
2075 // The run listener skips reconcile commands, so the recovery below finishes the task here.
2076 yield* eventSink.write({
2077 commandId: CommandId.make("command:runtime-reconcile:long-parent"),
2078 events: [
2079 {
2080 id: EventId.make("event:long-parent:child-completed"),
2081 type: "run.updated",
2082 threadId: childThreadId,
2083 runId: child.runs[0]!.id,
2084 occurredAt: now,
2085 payload: { ...child.runs[0]!, status: "completed", completedAt: now },
2086 },
2087 ],
2088 });
2089 yield* orchestrator.recoverDelegatedTasks.pipe(recordStatements);
2090 const finished = yield* projections.getThreadRecords(parentThreadId, [
2091 "subagents",
2092 "nodes",
2093 "turnItems",
2094 "runs",
2095 ]);
2096 assert.deepEqual(
2097 [
2098 finished.subagents[0]?.status,
2099 finished.subagents[0]?.completionDelivery?.state,
2100 finished.nodes.find((node) => node.id === task.id)?.status,
2101 finished.turnItems.find((item) => item.type === "subagent")?.status,
2102 finished.runs[0]?.delegatedCompletion?.delivery?.taskIds,
2103 ],
2104 ["completed", "claimed", "completed", "completed", [task.id]],
2105 );
2106
2107 // A person's follow-up to the finished child binds a new task to the parent's running run.
2108 yield* orchestrator
2109 .dispatch({
2110 type: "message.dispatch",
2111 commandId: CommandId.make("follow-up-long-parent"),
2112 threadId: childThreadId,
2113 messageId: MessageId.make("message:long-parent:follow-up"),
2114 text: "One more thing",
2115 attachments: [],
2116 dispatchMode: { type: "start_immediately" },
2117 createdBy: "user",
2118 creationSource: "web",
2119 })
2120 .pipe(Effect.withSpan("test.followUp"), recordStatements);
2121 const followedUp = yield* projections.getThreadRecords(parentThreadId, [
2122 "subagents",
2123 "turnItems",
2124 ]);
2125 const followUp = followedUp.subagents.find((candidate) => candidate.id !== task.id);
2126 assert.deepEqual(
2127 [
2128 followUp?.runId,
2129 followedUp.turnItems.find(
2130 (item) => item.type === "subagent" && item.subagentId === followUp?.id,
2131 )?.providerTurnId,
2132 ],
2133 [parentRun.id, providerTurnId],
2134 );
2135
2136 // Start, finish and follow-up read the task's own records and the run it binds to, so their VM
2137 // steps, which are deterministic, must stay flat on a copy with a long parent history. Left out:
2138 // the counts that number the task and its wake, which walk one index.
2139 const copyPath = "file:/delegated-task-reads?vfs=memdb";
2140 using copy = new NodeSqlite.DatabaseSync(copyPath);
2141 yield* sql`VACUUM INTO ${copyPath}`;
2142 const reads = statements.filter(
2143 ([query]) => /^\s*(SELECT|WITH)\b/.test(query) && !query.includes("COUNT(*)"),
2144 );
2145 const readSteps = () =>
2146 reads.reduce((steps, [query, params]) => {
2147 const statement = copy.prepare(query);
2148 statement.all(...(params as ReadonlyArray<NodeSqlite.SQLInputValue>));
2149 return steps + statement.stat("vmStep");
2150 }, 0);
2151 const shortSteps = readSteps();
2152 assert.isAbove(shortSteps, 0);
2153 const addHistory = (table: string, key: string, renamed: ReadonlyArray<string> = []) => {
2154 const name = `orchestration_v2_projection_${table}`;
2155 const columns = copy
2156 .prepare(`SELECT name FROM pragma_table_info('${name}')`)
2157 .all()
2158 .map((column) => String(column.name));
2159 const values = columns.map((column) =>
2160 column === key || renamed.includes(column)
2161 ? `CASE WHEN ${column} = '${parentThreadId}' THEN ${column} ELSE ${column} || ':history:' || history.n END`
2162 : column === "ordinal"
2163 ? "ordinal + history.n"
2164 : column,
2165 );
2166 copy.exec(`WITH RECURSIVE history(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM history WHERE n < 300)
2167 INSERT INTO ${name} (${columns.join(", ")})
2168 SELECT ${values.join(", ")} FROM ${name}, history
2169 WHERE '${parentThreadId}' IN (${columns.filter((column) => column.endsWith("thread_id")).join(", ")})`);
2170 };
2171 addHistory("nodes", "node_id");
2172 addHistory("turn_items", "turn_item_id", ["node_id"]);
2173 addHistory("subagents", "subagent_id", ["child_thread_id"]);
2174 addHistory("context_transfers", "context_transfer_id", [
2175 "source_thread_id",
2176 "target_thread_id",
2177 ]);
2178 addHistory("runs", "run_id");
2179 // The history's runs have ended; only the parent's own run is still going.
2180 copy.exec(`UPDATE orchestration_v2_projection_runs
2181 SET status = 'completed', completed_at = requested_at WHERE run_id LIKE '%:history:%'`);
2182 addHistory("run_attempts", "attempt_id", ["run_id"]);
2183 addHistory("messages", "message_id");
2184 addHistory("provider_turns", "provider_turn_id", ["run_attempt_id"]);
2185 assert.isAtMost(readSteps(), shortSteps * 1.1);
2186 }).pipe(Effect.provide(layerTest)),
2187 );
2188
2189 it.effect("records a secret and a link card without reading the thread's history", () =>
2190 Effect.gen(function* () {
2191 const orchestrator = yield* Orchestrator.OrchestratorV2;
2192 const projections = yield* ProjectionStore.ProjectionStoreV2;
2193 const sql = yield* SqlClient.SqlClient;
2194 const threadId = ThreadId.make("thread:long-cards");
2195 const now = yield* DateTime.now;
2196 yield* orchestrator.dispatch({
2197 type: "thread.create",
2198 commandId: CommandId.make("create-long-cards"),
2199 threadId,
2200 projectId: ProjectId.make("project:long-cards"),
2201 title: "Cards",
2202 modelSelection,
2203 runtimeMode: "full-access",
2204 interactionMode: "default",
2205 branch: null,
2206 worktreePath: null,
2207 createdBy: "user",
2208 creationSource: "web",
2209 });
2210 yield* orchestrator.dispatch({
2211 type: "message.dispatch",
2212 commandId: CommandId.make("start-long-cards"),
2213 threadId,
2214 messageId: MessageId.make("message:long-cards"),
2215 text: "Ask for a token and a link",
2216 attachments: [],
2217 dispatchMode: { type: "start_immediately" },
2218 createdBy: "user",
2219 creationSource: "web",
2220 });
2221 const run = (yield* projections.getThreadRecords(threadId, ["runs"])).runs[0]!;
2222 const providerTurnId = ProviderTurnId.make("provider-turn:long-cards");
2223 yield* projections.apply({
2224 id: EventId.make("event:long-cards:provider-turn"),
2225 type: "provider-turn.updated",
2226 threadId,
2227 runId: run.id,
2228 occurredAt: now,
2229 payload: {
2230 id: providerTurnId,
2231 providerThreadId: run.providerThreadId!,
2232 nodeId: run.rootNodeId!,
2233 runAttemptId: run.activeAttemptId,
2234 nativeTurnRef: null,
2235 ordinal: 1,
2236 status: "running",
2237 startedAt: now,
2238 completedAt: null,
2239 },
2240 });
2241
2242 const cardSpans = [
2243 "orchestrationV2.dispatch.secretRequestRecord",
2244 "orchestrationV2.dispatch.linkRequestRecord",
2245 ];
2246 const statements: Array<readonly [string, ReadonlyArray<unknown>]> = [];
2247 const recordStatements = Effect.provideService(
2248 Statement.CurrentTransformer,
2249 (statement, _make, _fiber, span) => {
2250 for (
2251 let parent = Option.some<Tracer.AnySpan>(span);
2252 Option.isSome(parent) && parent.value._tag === "Span";
2253 parent = parent.value.parent
2254 ) {
2255 if (cardSpans.includes(parent.value.name)) {
2256 statements.push(statement.compile());
2257 break;
2258 }
2259 }
2260 return Effect.succeed(statement);
2261 },
2262 );
2263 const card = {
2264 threadId,
2265 runId: run.id,
2266 nodeId: run.rootNodeId!,
2267 reason: "Needed for the work",
2268 };
2269 yield* orchestrator
2270 .dispatch({
2271 ...card,
2272 type: "secret_request.record",
2273 commandId: CommandId.make("record-long-cards:secret"),
2274 turnItemId: TurnItemId.make("item:long-cards:secret"),
2275 label: "Token",
2276 secretStatus: "pending",
2277 })
2278 .pipe(recordStatements);
2279 yield* orchestrator
2280 .dispatch({
2281 ...card,
2282 type: "link_request.record",
2283 commandId: CommandId.make("record-long-cards:link"),
2284 turnItemId: TurnItemId.make("item:long-cards:link"),
2285 url: "https://example.test",
2286 linkStatus: "pending",
2287 })
2288 .pipe(recordStatements);
2289 const { turnItems } = yield* projections.getThreadRecords(threadId, ["turnItems"], {
2290 turnItemTypes: ["secret_request", "link_request"],
2291 });
2292 assert.deepEqual(
2293 turnItems.map((item) => [item.type, item.providerTurnId]),
2294 [
2295 ["secret_request", providerTurnId],
2296 ["link_request", providerTurnId],
2297 ],
2298 );
2299
2300 // A card reads its own run, so the rows and VM steps of its reads, which are deterministic,
2301 // must stay flat on a copy where the thread has thousands of older runs.
2302 const copyPath = "file:/card-reads?vfs=memdb";
2303 using copy = new NodeSqlite.DatabaseSync(copyPath);
2304 yield* sql`VACUUM INTO ${copyPath}`;
2305 const replay = () =>
2306 statements
2307 .filter(([query]) => /^\s*(SELECT|WITH)\b/.test(query))
2308 .reduce(
2309 (total, [query, params]) => {
2310 const statement = copy.prepare(query);
2311 const rows = statement.all(...(params as ReadonlyArray<NodeSqlite.SQLInputValue>));
2312 return {
2313 rows: total.rows + rows.length,
2314 steps: total.steps + statement.stat("vmStep"),
2315 };
2316 },
2317 { rows: 0, steps: 0 },
2318 );
2319 const short = replay();
2320 const history: ReadonlyArray<readonly [string, string, ReadonlyArray<string>]> = [
2321 ["runs", "run_id", []],
2322 ["run_attempts", "attempt_id", ["run_id"]],
2323 ["nodes", "node_id", ["run_id"]],
2324 ["provider_turns", "provider_turn_id", ["run_attempt_id"]],
2325 ];
2326 for (const [table, key, renamed] of history) {
2327 const name = `orchestration_v2_projection_${table}`;
2328 const columns = copy
2329 .prepare(`SELECT name FROM pragma_table_info('${name}')`)
2330 .all()
2331 .map((column) => String(column.name));
2332 const values = columns.map((column) =>
2333 column === key || renamed.includes(column)
2334 ? `${column} || ':history:' || history.n`
2335 : column === "ordinal"
2336 ? "ordinal + history.n"
2337 : column,
2338 );
2339 copy.exec(`WITH RECURSIVE history(n) AS (SELECT 1 UNION ALL SELECT n + 1 FROM history WHERE n < 2000)
2340 INSERT INTO ${name} (${columns.join(", ")})
2341 SELECT ${values.join(", ")} FROM ${name}, history WHERE thread_id = '${threadId}'`);
2342 }
2343 const long = replay();
2344 assert.equal(long.rows, short.rows);
2345 assert.isAtMost(long.steps, short.steps * 1.1);
2346 }).pipe(Effect.provide(layerTest)),
2347 );
2348
2349 it.effect("replays a thread.create receipt without changing the thread", () =>
2350 Effect.gen(function* () {
2351 const orchestrator = yield* Orchestrator.OrchestratorV2;
2352 const projections = yield* ProjectionStore.ProjectionStoreV2;
2353 const threadId = ThreadId.make("thread:create-receipt-replay");
2354 const create = {
2355 type: "thread.create",
2356 commandId: CommandId.make("create:receipt-replay"),
2357 threadId,
2358 projectId: ProjectId.make("project:duplicate-create"),
2359 title: "Original",
2360 modelSelection,
2361 runtimeMode: "full-access",
2362 interactionMode: "default",
2363 branch: null,
2364 worktreePath: null,
2365 createdBy: "user",
2366 creationSource: "web",
2367 } as const;
2368 const original = yield* orchestrator.dispatch(create);
2369 const before = yield* projections.getThread(threadId);
2370 const replayed = yield* orchestrator.dispatch(create);
2371 assert.equal(replayed.sequence, original.sequence);
2372 // Marked so client analytics do not count the retry again.
2373 assert.deepEqual(replayed, { ...original, replayed: true });
2374 assert.deepEqual(yield* projections.getThread(threadId), before);
2375 assert.equal(yield* orchestrator.getThreadEventSequence(threadId), original.sequence);
2376 }).pipe(Effect.provide(layerTest)),
2377 );
2378
2379 it.effect("explicit settle clears snooze atomically and stays idempotent after re-snoozing", () =>
2380 Effect.gen(function* () {
2381 const orchestrator = yield* Orchestrator.OrchestratorV2;
2382 const projections = yield* ProjectionStore.ProjectionStoreV2;
2383 const events = yield* EventSink.EventSinkV2;
2384 const threadId = ThreadId.make("thread:settle-snoozed");
2385 yield* orchestrator.dispatch({
2386 type: "thread.create",
2387 commandId: CommandId.make("create-settle-snoozed"),
2388 threadId,
2389 projectId: ProjectId.make("project:settle-snoozed"),
2390 title: "Snoozed thread",
2391 modelSelection,
2392 runtimeMode: "full-access",
2393 interactionMode: "default",
2394 branch: null,
2395 worktreePath: null,
2396 createdBy: "user",
2397 creationSource: "web",
2398 });
2399 const now = yield* DateTime.now;
2400 const settledAt = DateTime.add(now, { seconds: 1 });
2401 const snoozedUntil = DateTime.add(now, { hours: 1 });
2402 for (const round of [1, 2]) {
2403 yield* orchestrator.dispatch({
2404 type: "thread.snooze",
2405 commandId: CommandId.make(`snooze-before-settle-${round}`),
2406 threadId,
2407 snoozedUntil: DateTime.formatIso(snoozedUntil),
2408 });
2409 assert.deepInclude(yield* projections.getThread(threadId), {
2410 snoozedUntil,
2411 snoozedAt: yield* DateTime.now,
2412 });
2413 yield* TestClock.adjust("1 second");
2414 const commandId = CommandId.make(`settle-snoozed-${round}`);
2415 yield* orchestrator.dispatch({ type: "thread.settle", commandId, threadId });
2416 const expected = {
2417 settledOverride: "settled",
2418 settledAt,
2419 snoozedUntil: null,
2420 snoozedAt: null,
2421 updatedAt: yield* DateTime.now,
2422 } as const;
2423 const committed = yield* events.readByCommandId({ commandId }).pipe(Stream.runCollect);
2424 assert.lengthOf(committed, 1);
2425 assert.equal(committed[0]?.event.type, "thread.settled");
2426 assert.deepInclude(committed[0]?.event.payload, expected);
2427 const settled = yield* projections.getThreadShell(threadId);
2428 assert.deepInclude(settled, expected);
2429 assert.deepInclude((yield* projections.getThreadProjection(threadId)).thread, expected);
2430
2431 yield* TestClock.adjust("1 second");
2432 yield* orchestrator.dispatch({
2433 type: "thread.settle",
2434 commandId: CommandId.make(`settle-again-${round}`),
2435 threadId,
2436 });
2437 assert.deepEqual(yield* projections.getThreadShell(threadId), settled);
2438 }
2439 yield* orchestrator.dispatch({
2440 type: "thread.unsettle",
2441 commandId: CommandId.make("unsettle-after-snooze"),
2442 threadId,
2443 reason: "user",
2444 });
2445 const active = {
2446 settledOverride: "active",
2447 settledAt: null,
2448 snoozedUntil: null,
2449 snoozedAt: null,
2450 } as const;
2451 assert.deepInclude(yield* projections.getThreadShell(threadId), active);
2452 assert.deepInclude((yield* projections.getThreadProjection(threadId)).thread, active);
2453
2454 // Undo of a settle restores the cleared Active anchor instead of stamping a new one.
2455 for (const restoreUnsettledAt of [DateTime.formatIso(now), null]) {
2456 yield* TestClock.adjust("1 second");
2457 yield* orchestrator.dispatch({
2458 type: "thread.settle",
2459 commandId: CommandId.make(`settle-before-undo-${restoreUnsettledAt}`),
2460 threadId,
2461 });
2462 yield* orchestrator.dispatch({
2463 type: "thread.unsettle",
2464 commandId: CommandId.make(`undo-settle-${restoreUnsettledAt}`),
2465 threadId,
2466 reason: "user",
2467 restoreUnsettledAt,
2468 });
2469 assert.deepInclude(yield* projections.getThreadShell(threadId), {
2470 ...active,
2471 unsettledAt: restoreUnsettledAt === null ? null : now,
2472 });
2473 }
2474 }).pipe(Effect.provide(layerTest)),
2475 );
2476
2477 it.effect("keeps a snooze when a live question is cancelled or an approval is answered", () =>
2478 Effect.gen(function* () {
2479 const orchestrator = yield* Orchestrator.OrchestratorV2;
2480 const projections = yield* ProjectionStore.ProjectionStoreV2;
2481 const outbox = yield* EffectOutbox.EffectOutboxV2;
2482 const now = yield* DateTime.now;
2483 for (const kind of ["user_input", "command"] as const) {
2484 const threadId = ThreadId.make(`thread:snoozed-live-${kind}`);
2485 const requestId = RuntimeRequestId.make(`request:snoozed-live-${kind}`);
2486 const sessionId = ProviderSessionId.make(`session:snoozed-live-${kind}`);
2487 yield* orchestrator.dispatch({
2488 type: "thread.create",
2489 commandId: CommandId.make(`create-snoozed-live-${kind}`),
2490 threadId,
2491 projectId: ProjectId.make("project:snoozed-live"),
2492 title: "Deferred question",
2493 modelSelection,
2494 runtimeMode: "full-access",
2495 interactionMode: "default",
2496 branch: null,
2497 worktreePath: null,
2498 createdBy: "user",
2499 creationSource: "web",
2500 });
2501 yield* orchestrator.dispatch({
2502 type: "thread.snooze",
2503 commandId: CommandId.make(`snooze-live-${kind}`),
2504 threadId,
2505 snoozedUntil: "2099-07-25T09:00:00.000Z",
2506 });
2507 const before = yield* projections.getThread(threadId);
2508 yield* projections.apply({
2509 id: EventId.make(`attach-snoozed-live-${kind}`),
2510 type: "provider-session.attached",
2511 threadId,
2512 occurredAt: now,
2513 payload: {
2514 id: sessionId,
2515 driver: adapter.driver,
2516 providerInstanceId: instanceId,
2517 status: "ready",
2518 cwd: "/repo",
2519 model: modelSelection.model,
2520 capabilities: CodexProviderCapabilitiesV2,
2521 createdAt: now,
2522 updatedAt: now,
2523 lastError: null,
2524 },
2525 });
2526 yield* projections.apply({
2527 id: EventId.make(`request-snoozed-live-${kind}`),
2528 type: "runtime-request.updated",
2529 threadId,
2530 occurredAt: now,
2531 payload: {
2532 id: requestId,
2533 nodeId: NodeId.make(`node:snoozed-live-${kind}`),
2534 providerTurnId: null,
2535 nativeRequestRef: null,
2536 kind,
2537 status: "pending",
2538 responseCapability: { type: "live", providerSessionId: sessionId },
2539 createdAt: now,
2540 resolvedAt: null,
2541 },
2542 });
2543 const commandId = CommandId.make(`respond-snoozed-live-${kind}`);
2544 yield* orchestrator.dispatch({
2545 type: "runtime-request.respond",
2546 commandId,
2547 threadId,
2548 requestId,
2549 decision: kind === "user_input" ? "cancel" : "accept",
2550 });
2551 const after = yield* projections.getThread(threadId);
2552 assert.deepEqual(after.snoozedUntil, before.snoozedUntil);
2553 assert.deepEqual(after.snoozedAt, before.snoozedAt);
2554 assert.equal((yield* projections.getRuntimeRequest(threadId, requestId))?.status, "resolved");
2555 assert.equal(
2556 (yield* outbox.listByCommandId(commandId))[0]?.request.type,
2557 "runtime-request.respond",
2558 );
2559 }
2560 }).pipe(Effect.provide(layerTest)),
2561 );
2562
2563 it.effect("dispatch records the command outcome and its duration", () =>
2564 Effect.gen(function* () {
2565 const orchestrator = yield* Orchestrator.OrchestratorV2;
2566 const successes = Metric.value(
2567 Metric.withAttributes(orchestrationCommandsTotal, {
2568 commandType: "thread.create",
2569 outcome: "success",
2570 }),
2571 );
2572 const durations = Metric.value(
2573 Metric.withAttributes(orchestrationCommandDuration, { commandType: "thread.create" }),
2574 );
2575 const successesBefore = (yield* successes).count;
2576 const durationsBefore = (yield* durations).count;
2577
2578 yield* orchestrator.dispatch({
2579 type: "thread.create",
2580 commandId: CommandId.make("create-dispatch-metrics"),
2581 threadId: ThreadId.make("thread:dispatch-metrics"),
2582 projectId: ProjectId.make("project:dispatch-metrics"),
2583 title: "Dispatch metrics",
2584 modelSelection,
2585 runtimeMode: "full-access",
2586 interactionMode: "default",
2587 branch: null,
2588 worktreePath: null,
2589 createdBy: "user",
2590 creationSource: "web",
2591 });
2592
2593 assert.strictEqual((yield* successes).count - successesBefore, 1);
2594 assert.strictEqual((yield* durations).count - durationsBefore, 1);
2595 }).pipe(Effect.provide(layerTest)),
2596 );
2597
2598 it.effect("a user message dispatched first stays run 1 when its lineage read is slow", () => {
2599 const threadId = ThreadId.make("thread:dispatch-order");
2600 // A file database serves projection reads from a reader worker, so the lineage read a user
2601 // dispatch makes before taking the thread lock can yield. This gate holds it open instead.
2602 const lineageReadEntered = Deferred.makeUnsafe<void>();
2603 const lineageReadRelease = Deferred.makeUnsafe<void>();
2604 let gateArmed = false;
2605 const layerGatedLineageRead = ProviderReplayHarness.layerWithRegistry(
2606 { name: "dispatch-order" },
2607 ProviderAdapterRegistry.layerFromAdapters([adapter]),
2608 {
2609 runEffectWorker: false,
2610 wrapProjectionStore: (store) => ({
2611 ...store,
2612 getThread: (id) => {
2613 if (!gateArmed || id !== threadId) return store.getThread(id);
2614 gateArmed = false;
2615 return Deferred.succeed(lineageReadEntered, undefined).pipe(
2616 Effect.andThen(Deferred.await(lineageReadRelease)),
2617 Effect.andThen(store.getThread(id)),
2618 );
2619 },
2620 }),
2621 },
2622 );
2623 return Effect.gen(function* () {
2624 const orchestrator = yield* Orchestrator.OrchestratorV2;
2625 yield* orchestrator.dispatch({
2626 type: "thread.create",
2627 commandId: CommandId.make("create-dispatch-order"),
2628 threadId,
2629 projectId: ProjectId.make("project:dispatch-order"),
2630 title: "Dispatch order",
2631 modelSelection,
2632 runtimeMode: "full-access",
2633 interactionMode: "default",
2634 branch: null,
2635 worktreePath: null,
2636 createdBy: "user",
2637 creationSource: "web",
2638 });
2639 const message = (id: string, createdBy: "user" | "agent") =>
2640 orchestrator.dispatch({
2641 type: "message.dispatch",
2642 commandId: CommandId.make(id),
2643 threadId,
2644 messageId: MessageId.make(id),
2645 text: id,
2646 attachments: [],
2647 dispatchMode: { type: "defer_start" },
2648 createdBy,
2649 creationSource: createdBy === "user" ? "web" : "server",
2650 });
2651
2652 gateArmed = true;
2653 const user = yield* message("message:user-first", "user").pipe(Effect.forkChild);
2654 yield* Deferred.await(lineageReadEntered);
2655 // Agent messages skip the lineage read, so this one would take a free thread lock first.
2656 const agent = yield* message("message:agent-second", "agent").pipe(
2657 Effect.forkChild({ startImmediately: true }),
2658 );
2659 yield* Deferred.succeed(lineageReadRelease, undefined);
2660 yield* Fiber.join(agent);
2661 yield* Fiber.join(user);
2662
2663 const { runs } = yield* orchestrator.getThreadProjection(threadId);
2664 assert.deepEqual(
2665 runs.toSorted((left, right) => left.ordinal - right.ordinal).map((run) => run.userMessageId),
2666 [MessageId.make("message:user-first"), MessageId.make("message:agent-second")],
2667 );
2668 }).pipe(Effect.provide(layerGatedLineageRead));
2669 });
2670
2671 it.effect("queued-message commands read a long thread's history without walking all of it", () =>
2672 Effect.gen(function* () {
2673 const orchestrator = yield* Orchestrator.OrchestratorV2;
2674 const projections = yield* ProjectionStore.ProjectionStoreV2;
2675 const sql = yield* SqlClient.SqlClient;
2676 const threadId = ThreadId.make("thread:long-history-reads");
2677 const driver = ProviderDriverKind.make("codex");
2678 const now = yield* DateTime.now;
2679 yield* orchestrator.dispatch({
2680 type: "thread.create",
2681 commandId: CommandId.make("create-long-history-reads"),
2682 threadId,
2683 projectId: ProjectId.make("project:long-history-reads"),
2684 title: "Long history",
2685 modelSelection,
2686 runtimeMode: "full-access",
2687 interactionMode: "default",
2688 branch: null,
2689 worktreePath: null,
2690 createdBy: "user",
2691 creationSource: "web",
2692 });
2693 const providerThreadId = ProviderThreadId.make("provider-thread:long-history-reads");
2694 yield* projections.apply({
2695 id: EventId.make("event:long-history-reads:provider-thread"),
2696 type: "provider-thread.updated",
2697 threadId,
2698 driver,
2699 providerInstanceId: instanceId,
2700 occurredAt: now,
2701 payload: {
2702 id: providerThreadId,
2703 appThreadId: threadId,
2704 ownerNodeId: null,
2705 driver,
2706 providerInstanceId: instanceId,
2707 providerSessionId: null,
2708 nativeThreadRef: null,
2709 nativeConversationHeadRef: null,
2710 status: "idle",
2711 firstRunOrdinal: 1,
2712 lastRunOrdinal: 4,
2713 handoffIds: [],
2714 forkedFrom: null,
2715 createdAt: now,
2716 updatedAt: now,
2717 pendingBackgroundTasks: [],
2718 },
2719 });
2720 const dispatch = (name: string) =>
2721 orchestrator.dispatch({
2722 type: "message.dispatch",
2723 commandId: CommandId.make(`dispatch:long-history-reads:${name}`),
2724 threadId,
2725 messageId: MessageId.make(`message:long-history-reads:${name}`),
2726 // A compacting target lets promote skip it without a live provider session.
2727 text: name === "active" ? "/compact" : `Request ${name}`,
2728 attachments: [],
2729 dispatchMode: { type: "defer_start" },
2730 createdBy: "user",
2731 creationSource: "web",
2732 });
2733 const nextOrdinal = Effect.gen(function* () {
2734 const [row] = yield* sql<{ readonly ordinal: number }>`
2735 SELECT COALESCE(MAX(ordinal), 0) + 1 AS ordinal
2736 FROM orchestration_v2_projection_turn_items WHERE thread_id = ${threadId}`;
2737 return row!.ordinal;
2738 });
2739 // Settled history: each run leaves completed leaf nodes, tool output and a reply.
2740 for (const name of ["1", "2", "3", "active"]) {
2741 yield* dispatch(name);
2742 const run = (yield* projections.getThreadProjection(threadId)).runs.at(-1)!;
2743 const active = name === "active";
2744 yield* projections.apply({
2745 id: EventId.make(`event:long-history-reads:${name}:run`),
2746 type: "run.updated",
2747 threadId,
2748 runId: run.id,
2749 occurredAt: now,
2750 payload: {
2751 ...run,
2752 providerThreadId,
2753 status: active ? "running" : "completed",
2754 queuePosition: null,
2755 startedAt: now,
2756 completedAt: active ? null : now,
2757 },
2758 });
2759 if (active) continue;
2760 for (const kind of ["tool_call", "assistant_message"] as const) {
2761 const nodeId = NodeId.make(`node:long-history-reads:${name}:${kind}`);
2762 yield* projections.apply({
2763 id: EventId.make(`event:${nodeId}`),
2764 type: "node.updated",
2765 threadId,
2766 runId: run.id,
2767 nodeId,
2768 driver,
2769 occurredAt: now,
2770 payload: {
2771 id: nodeId,
2772 threadId,
2773 runId: run.id,
2774 rootNodeId: run.rootNodeId!,
2775 parentNodeId: run.rootNodeId,
2776 kind,
2777 status: "completed",
2778 countsForRun: false,
2779 providerThreadId,
2780 providerTurnId: null,
2781 nativeItemRef: null,
2782 runtimeRequestId: null,
2783 checkpointScopeId: null,
2784 startedAt: now,
2785 completedAt: now,
2786 },
2787 });
2788 }
2789 const replyId = MessageId.make(`message:long-history-reads:${name}:reply`);
2790 yield* projections.apply({
2791 id: EventId.make(`event:${replyId}`),
2792 type: "message.updated",
2793 threadId,
2794 occurredAt: now,
2795 payload: {
2796 id: replyId,
2797 threadId,
2798 runId: run.id,
2799 nodeId: NodeId.make(`node:long-history-reads:${name}:assistant_message`),
2800 createdBy: "agent",
2801 creationSource: "server",
2802 role: "assistant",
2803 text: `Reply ${name}`,
2804 attachments: [],
2805 streaming: false,
2806 createdAt: now,
2807 updatedAt: now,
2808 },
2809 });
2810 const itemBase = {
2811 threadId,
2812 runId: run.id,
2813 providerThreadId,
2814 providerTurnId: null,
2815 nativeItemRef: null,
2816 parentItemId: null,
2817 status: "completed" as const,
2818 startedAt: now,
2819 completedAt: now,
2820 updatedAt: now,
2821 };
2822 for (const item of [
2823 {
2824 ...itemBase,
2825 id: TurnItemId.make(`item:long-history-reads:${name}:command`),
2826 nodeId: NodeId.make(`node:long-history-reads:${name}:tool_call`),
2827 ordinal: yield* nextOrdinal,
2828 title: "command",
2829 type: "command_execution" as const,
2830 input: "cat build.log",
2831 },
2832 {
2833 ...itemBase,
2834 id: TurnItemId.make(`item:long-history-reads:${name}:reply`),
2835 nodeId: NodeId.make(`node:long-history-reads:${name}:assistant_message`),
2836 ordinal: yield* nextOrdinal,
2837 title: null,
2838 type: "assistant_message" as const,
2839 messageId: replyId,
2840 text: `Reply ${name}`,
2841 streaming: false,
2842 },
2843 ]) {
2844 yield* projections.apply({
2845 id: EventId.make(`event:${item.id}`),
2846 type: "turn-item.updated",
2847 threadId,
2848 runId: run.id,
2849 nodeId: item.nodeId,
2850 driver,
2851 occurredAt: now,
2852 payload: item,
2853 });
2854 }
2855 }
2856 const activeRun = (yield* projections.getThreadProjection(threadId)).runs.at(-1)!;
2857 // Stop needs a live provider turn to interrupt.
2858 yield* projections.apply({
2859 id: EventId.make("event:long-history-reads:active-turn"),
2860 type: "provider-turn.updated",
2861 threadId,
2862 runId: activeRun.id,
2863 occurredAt: now,
2864 payload: {
2865 id: ProviderTurnId.make("provider-turn:long-history-reads"),
2866 providerThreadId,
2867 nodeId: activeRun.rootNodeId!,
2868 runAttemptId: activeRun.activeAttemptId!,
2869 nativeTurnRef: { driver, nativeId: "native:long-history-reads", strength: "strong" },
2870 ordinal: 4,
2871 status: "running",
2872 startedAt: now,
2873 completedAt: null,
2874 },
2875 });
2876
2877 const statements: Array<ReturnType<Statement.Statement<unknown>["compile"]>> = [];
2878 const observe: Statement.Transformer = (statement) => {
2879 const compiled = statement.compile();
2880 // Row reads and the message-ordering pass; a MAX(ordinal) probe reads one index entry.
2881 if (/_(nodes|turn_items)\b/.test(compiled[0]) && /payload_json|GROUP BY/.test(compiled[0])) {
2882 statements.push(compiled);
2883 }
2884 return Effect.succeed(statement);
2885 };
2886 const historyWalks = Effect.fnUntraced(function* (commandType: string) {
2887 const walks: Array<string> = [];
2888 for (const [query, params] of statements.splice(0)) {
2889 const plan = yield* sql.unsafe<{ detail: string }>(`EXPLAIN QUERY PLAN ${query}`, params);
2890 for (const { detail } of plan) {
2891 // A history-sized read: an ordering pass over every message item for a narrowed
2892 // message read, or a node/turn-item row search bounded by nothing but the thread.
2893 // Walking only a covering index's ids reads no payloads. Stop still loads every
2894 // message, so its ordering pass is no larger than that read.
2895 if (
2896 (detail.includes("TEMP B-TREE FOR GROUP BY") && query.includes("json_each")) ||
2897 (/^SEARCH .*(node|turn_item).* \(thread_id=\?\)$/.test(detail) &&
2898 !detail.includes("USING COVERING INDEX") &&
2899 // That full ordering pass, through an index that holds only message items.
2900 !(detail.includes("message_ordinal_idx") && !query.includes("json_each"))) ||
2901 // Every active run in the environment, not only this thread's.
2902 /^SEARCH .*runs.* \(status=\?\)$/.test(detail) ||
2903 // A status-filtered read that fetches every typed row to check its status.
2904 (/^SEARCH .*turn_items USING INDEX .* \(thread_id=\? AND type=\?\)$/.test(detail) &&
2905 query.includes("status IN"))
2906 ) {
2907 walks.push(`${commandType}: ${detail}\n in: ${query.replace(/\s+/g, " ").trim()}`);
2908 }
2909 }
2910 }
2911 return walks;
2912 });
2913 const observed = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
2914 effect.pipe(Effect.provideService(Statement.CurrentTransformer, observe));
2915
2916 yield* observed(dispatch("queued"));
2917 const walks = yield* historyWalks("message.dispatch");
2918 const queuedRun = (yield* projections.getThreadProjection(threadId)).runs.at(-1)!;
2919 assert.equal(queuedRun.status, "queued");
2920 yield* observed(
2921 orchestrator.dispatch({
2922 type: "queued-run.edit",
2923 commandId: CommandId.make("edit:long-history-reads"),
2924 threadId,
2925 runId: queuedRun.id,
2926 text: "Request queued, edited",
2927 }),
2928 );
2929 walks.push(...(yield* historyWalks("queued-run.edit")));
2930 yield* observed(
2931 orchestrator.dispatch({
2932 type: "queued-message.promote-to-steer",
2933 commandId: CommandId.make("promote:long-history-reads"),
2934 threadId,
2935 queuedRunId: queuedRun.id,
2936 targetRunId: activeRun.id,
2937 }),
2938 );
2939 walks.push(...(yield* historyWalks("queued-message.promote-to-steer")));
2940 yield* dispatch("cancelled");
2941 const cancelledRun = (yield* projections.getThreadProjection(threadId)).runs.at(-1)!;
2942 assert.equal(cancelledRun.status, "queued");
2943 statements.splice(0);
2944 yield* observed(
2945 orchestrator.dispatch({
2946 type: "queued-run.cancel",
2947 commandId: CommandId.make("cancel:long-history-reads"),
2948 threadId,
2949 runId: cancelledRun.id,
2950 }),
2951 );
2952 walks.push(...(yield* historyWalks("queued-run.cancel")));
2953 yield* observed(
2954 orchestrator.dispatch({
2955 type: "run.interrupt",
2956 commandId: CommandId.make("interrupt:long-history-reads"),
2957 threadId,
2958 runId: activeRun.id,
2959 }),
2960 );
2961 walks.push(...(yield* historyWalks("run.interrupt")));
2962
2963 const after = yield* projections.getThreadProjection(threadId);
2964 assert.deepEqual(
2965 after.messages.filter((message) => message.role === "user").map((message) => message.text),
2966 [
2967 "Request 1",
2968 "Request 2",
2969 "Request 3",
2970 "/compact",
2971 "Request cancelled",
2972 "Request queued, edited",
2973 ],
2974 );
2975 assert.deepEqual(
2976 after.providerThreads.map((providerThread) => providerThread.id),
2977 [providerThreadId],
2978 );
2979 assert.deepEqual(walks, [], walks.join("\n"));
2980 }).pipe(Effect.provide(layerTest)),
2981 );
2982
2983 it.effect("a running provider-turn update reads only the thread's queued and running runs", () =>
2984 Effect.gen(function* () {
2985 const runsReads = yield* Deferred.make<ReturnType<Statement.Statement<unknown>["compile"]>>();
2986 let armed = false;
2987 // The deferred-steer reactor runs in a fiber forked by the layer, so observe at layer level.
2988 const observe: Statement.Transformer = (statement) => {
2989 const compiled = statement.compile();
2990 return armed && /FROM orchestration_v2_projection_runs\b/.test(compiled[0])
2991 ? Deferred.succeed(runsReads, compiled).pipe(Effect.as(statement))
2992 : Effect.succeed(statement);
2993 };
2994 yield* Effect.gen(function* () {
2995 const orchestrator = yield* Orchestrator.OrchestratorV2;
2996 const projections = yield* ProjectionStore.ProjectionStoreV2;
2997 const eventSink = yield* EventSink.EventSinkV2;
2998 const sql = yield* SqlClient.SqlClient;
2999 const threadId = ThreadId.make("thread:deferred-steer-reads");
3000 const driver = ProviderDriverKind.make("codex");
3001 const now = yield* DateTime.now;
3002 yield* orchestrator.dispatch({
3003 type: "thread.create",
3004 commandId: CommandId.make("create-deferred-steer-reads"),
3005 threadId,
3006 projectId: ProjectId.make("project:deferred-steer-reads"),
3007 title: "Deferred steer reads",
3008 modelSelection,
3009 runtimeMode: "full-access",
3010 interactionMode: "default",
3011 branch: null,
3012 worktreePath: null,
3013 createdBy: "user",
3014 creationSource: "web",
3015 });
3016 yield* orchestrator.dispatch({
3017 type: "message.dispatch",
3018 commandId: CommandId.make("dispatch:deferred-steer-reads"),
3019 threadId,
3020 messageId: MessageId.make("message:deferred-steer-reads"),
3021 text: "Request",
3022 attachments: [],
3023 dispatchMode: { type: "defer_start" },
3024 createdBy: "user",
3025 creationSource: "web",
3026 });
3027 const run = (yield* projections.getThreadProjection(threadId)).runs.at(-1)!;
3028 const providerThreadId = ProviderThreadId.make("provider-thread:deferred-steer-reads");
3029 yield* eventSink.write({
3030 events: [
3031 {
3032 id: EventId.make("event:deferred-steer-reads:run"),
3033 type: "run.updated",
3034 threadId,
3035 runId: run.id,
3036 occurredAt: now,
3037 payload: { ...run, providerThreadId, status: "running", queuePosition: null },
3038 },
3039 ],
3040 });
3041 armed = true;
3042 yield* eventSink.write({
3043 events: [
3044 {
3045 id: EventId.make("event:deferred-steer-reads:turn"),
3046 type: "provider-turn.updated",
3047 threadId,
3048 runId: run.id,
3049 occurredAt: now,
3050 payload: {
3051 id: ProviderTurnId.make("provider-turn:deferred-steer-reads"),
3052 providerThreadId,
3053 nodeId: run.rootNodeId!,
3054 runAttemptId: run.activeAttemptId!,
3055 nativeTurnRef: {
3056 driver,
3057 nativeId: "native:deferred-steer-reads",
3058 strength: "strong",
3059 },
3060 ordinal: 1,
3061 status: "running",
3062 startedAt: now,
3063 completedAt: null,
3064 },
3065 },
3066 ],
3067 });
3068 const [query, params] = yield* Deferred.await(runsReads);
3069 const plan = yield* sql.unsafe<{ detail: string }>(`EXPLAIN QUERY PLAN ${query}`, params);
3070 // Every run the thread ever had, when only queued steers and the running run matter.
3071 assert.isFalse(
3072 plan.some(({ detail }) => /^SEARCH .*runs.* \(thread_id=\?\)$/.test(detail)),
3073 `${plan.map(({ detail }) => detail).join("\n")}\n in: ${query.replace(/\s+/g, " ").trim()}`,
3074 );
3075 }).pipe(
3076 Effect.provide(
3077 layerTest.pipe(Layer.provide(Layer.succeed(Statement.CurrentTransformer, observe))),
3078 ),
3079 );
3080 }),
3081 );
3082
3083 it.effect(
3084 "a delegated child's run end reads the parent's runs no matter how many deliveries it has open",
3085 () =>
3086 Effect.gen(function* () {
3087 const orchestrator = yield* Orchestrator.OrchestratorV2;
3088 const projections = yield* ProjectionStore.ProjectionStoreV2;
3089 const parentThreadId = ThreadId.make("thread:busy-coordinator");
3090 const now = yield* DateTime.now;
3091 yield* orchestrator.dispatch({
3092 type: "thread.create",
3093 commandId: CommandId.make("create-busy-coordinator"),
3094 threadId: parentThreadId,
3095 projectId: ProjectId.make("project:busy-coordinator"),
3096 title: "Coordinator",
3097 modelSelection,
3098 runtimeMode: "full-access",
3099 interactionMode: "default",
3100 branch: null,
3101 worktreePath: null,
3102 createdBy: "user",
3103 creationSource: "web",
3104 });
3105 yield* orchestrator.dispatch({
3106 type: "message.dispatch",
3107 commandId: CommandId.make("start-busy-coordinator"),
3108 threadId: parentThreadId,
3109 messageId: MessageId.make("message:busy-coordinator"),
3110 text: "Delegate the work",
3111 attachments: [],
3112 dispatchMode: { type: "start_immediately" },
3113 createdBy: "user",
3114 creationSource: "web",
3115 });
3116 const parentRun = (yield* projections.getThreadRecords(parentThreadId, ["runs"])).runs[0]!;
3117 yield* orchestrator.dispatch({
3118 type: "delegated_task.request",
3119 commandId: CommandId.make("delegate-busy-coordinator"),
3120 parentThreadId,
3121 parentRunId: parentRun.id,
3122 parentNodeId: parentRun.rootNodeId!,
3123 task: "Do the work",
3124 completionWake: "always",
3125 modelSelection,
3126 runtimeMode: "full-access",
3127 interactionMode: "default",
3128 createdBy: "agent",
3129 creationSource: "mcp",
3130 });
3131 const task = (yield* projections.getThreadRecords(parentThreadId, ["subagents"]))
3132 .subagents[0]!;
3133 const childThreadId = task.childThreadId!;
3134 const childRun = (yield* projections.getThreadRecords(childThreadId, ["runs"])).runs[0]!;
3135
3136 const runReads: Array<string> = [];
3137 const recordRunReads = Effect.provideService(
3138 Statement.CurrentTransformer,
3139 (statement, _make, _fiber, _span) => {
3140 const [query, params] = statement.compile();
3141 if (
3142 query.includes("FROM orchestration_v2_projection_runs") &&
3143 params.includes(parentThreadId)
3144 )
3145 runReads.push(query);
3146 return Effect.succeed(statement);
3147 },
3148 );
3149 const captureReads = Effect.gen(function* () {
3150 runReads.length = 0;
3151 yield* orchestrator
3152 .recoverDelegatedTask(childThreadId, { completedRunId: childRun.id })
3153 .pipe(recordRunReads);
3154 return runReads.length;
3155 });
3156 const quietParent = yield* captureReads;
3157
3158 // Completion wakes queued behind the coordinator's own work, each with a delivery still open.
3159 for (let index = 0; index < 40; index++) {
3160 const runId = RunId.make(`run:busy-coordinator:wake:${index}`);
3161 yield* projections.apply({
3162 id: EventId.make(`event:busy-coordinator:wake:${index}`),
3163 type: "run.updated",
3164 threadId: parentThreadId,
3165 runId,
3166 occurredAt: now,
3167 payload: {
3168 ...parentRun,
3169 id: runId,
3170 ordinal: parentRun.ordinal + index + 1,
3171 status: "queued",
3172 delegatedCompletion: {
3173 disposition: "open",
3174 nextGeneration: 2,
3175 delivery: {
3176 generation: 1,
3177 messageId: MessageId.make(`message:busy-coordinator:wake:${index}`),
3178 taskIds: [NodeId.make(`task:busy-coordinator:other:${index}`)],
3179 },
3180 },
3181 },
3182 });
3183 }
3184 assert.equal(yield* captureReads, quietParent);
3185 }).pipe(Effect.provide(layerTest)),
3186 );