The best way to control Claude Code, Codex and more from any computer or phone

main
ts 787 lines 29.2 KB
Raw
1 import type {
2 OrchestrationV2Command,
3 OrchestrationV2DomainEvent,
4 OrchestrationV2RuntimeRequest,
5 OrchestrationV2Run,
6 OrchestrationV2ThreadShellSnapshot,
7 OrchestrationV2StoredEvent,
8 OrchestrationV2ThreadProjection,
9 OrchestrationV2TurnItem,
10 ProviderApprovalDecision,
11 ProviderUserInputAnswers,
12 CommandId,
13 ThreadId,
14 TurnItemId,
15 } from "@t3tools/contracts";
16 import * as Clock from "effect/Clock";
17 import * as Deferred from "effect/Deferred";
18 import * as Duration from "effect/Duration";
19 import * as Effect from "effect/Effect";
20 import * as Fiber from "effect/Fiber";
21 import * as Option from "effect/Option";
22 import * as Ref from "effect/Ref";
23 import * as Schema from "effect/Schema";
24 import * as Stream from "effect/Stream";
25 import { TestClock } from "effect/testing";
26
27 import * as Orchestrator from "../Orchestrator.ts";
28 import type { RecordedCommandOutput } from "./CommandOutputRecorder.ts";
29 import type { ProviderReplayGate } from "@t3tools/provider-testing/replayGate";
30
31 export type OrchestratorV2ScenarioStep =
32 | {
33 readonly type: "dispatch";
34 readonly command: OrchestrationV2Command;
35 readonly await?: boolean;
36 readonly key?: string;
37 }
38 | {
39 readonly type: "advance_clock";
40 readonly duration: Duration.Input;
41 /** Wait until the jump idle-releases this thread's provider session. */
42 readonly awaitProviderSessionRelease?: ThreadId;
43 }
44 | {
45 readonly type: "await";
46 readonly key: string;
47 }
48 | {
49 readonly type: "await_all";
50 }
51 | {
52 readonly type: "await_thread_idle";
53 readonly threadId: ThreadId;
54 }
55 | {
56 readonly type: "await_no_background_work";
57 readonly threadId: ThreadId;
58 }
59 | {
60 readonly type: "await_run_steerable";
61 readonly threadId: ThreadId;
62 readonly runId: OrchestrationV2Run["id"];
63 }
64 | {
65 readonly type: "await_run_status";
66 readonly threadId: ThreadId;
67 readonly runId: OrchestrationV2Run["id"];
68 readonly status: OrchestrationV2Run["status"];
69 }
70 | {
71 readonly type: "finish_held_run";
72 readonly threadId: ThreadId;
73 readonly runId: OrchestrationV2Run["id"];
74 readonly status: OrchestrationV2Run["status"];
75 }
76 | {
77 readonly type: "await_run_turn_item";
78 readonly threadId: ThreadId;
79 readonly runId: OrchestrationV2Run["id"];
80 readonly itemType: OrchestrationV2TurnItem["type"];
81 /** Wait until the item also has this status. */
82 readonly itemStatus?: OrchestrationV2TurnItem["status"];
83 /** Look in the threads of the run's provider subagents instead of the run itself. */
84 readonly inSubagentThreads?: boolean;
85 }
86 | {
87 readonly type: "release_replay_gate_after_waiting";
88 readonly label: string;
89 readonly threadId: ThreadId;
90 readonly runId: OrchestrationV2Run["id"];
91 }
92 | {
93 readonly type: "release_replay_gate";
94 readonly label: string;
95 }
96 | {
97 readonly type: "capture_shell_snapshot";
98 readonly key: string;
99 }
100 | {
101 readonly type: "respond_to_next_runtime_request";
102 readonly threadId: ThreadId;
103 readonly commandId: CommandId;
104 readonly decision?: ProviderApprovalDecision;
105 readonly answers?: ProviderUserInputAnswers;
106 readonly shellSnapshotKeyWhilePending?: string;
107 };
108
109 export interface OrchestratorV2Scenario {
110 readonly name: string;
111 readonly commands: ReadonlyArray<OrchestrationV2Command>;
112 readonly steps?: ReadonlyArray<OrchestratorV2ScenarioStep>;
113 readonly projectionThreadIds?: ReadonlyArray<ThreadId>;
114 }
115
116 export interface OrchestratorV2ScenarioResult {
117 readonly storedEvents: ReadonlyArray<OrchestrationV2StoredEvent>;
118 readonly domainEvents: ReadonlyArray<OrchestrationV2DomainEvent>;
119 readonly projections: ReadonlyMap<ThreadId, OrchestrationV2ThreadProjection>;
120 readonly shellSnapshot: OrchestrationV2ThreadShellSnapshot;
121 readonly capturedShellSnapshots: ReadonlyMap<string, OrchestrationV2ThreadShellSnapshot>;
122 /** Live output each command streamed (never persisted). Provider replays fill it in. */
123 readonly commandOutput?: ReadonlyMap<TurnItemId, RecordedCommandOutput>;
124 }
125
126 export class OrchestratorV2ScenarioStepError extends Schema.TaggedError<OrchestratorV2ScenarioStepError>()(
127 "OrchestratorV2ScenarioStepError",
128 {
129 scenario: Schema.String,
130 step: Schema.String,
131 },
132 ) {
133 override get message(): string {
134 return `Invalid orchestrator scenario step ${this.step} in ${this.scenario}.`;
135 }
136 }
137
138 function commandThreadIds(command: OrchestrationV2Command): ReadonlyArray<ThreadId> {
139 switch (command.type) {
140 case "thread.create":
141 case "thread.stop":
142 case "thread.archive":
143 case "thread.unarchive":
144 case "thread.delete":
145 case "thread.guest-share.close":
146 case "thread.guest-message.dismiss":
147 case "thread.settle":
148 case "thread.auto-settle":
149 case "thread.unsettle":
150 case "thread.snooze":
151 case "thread.unsnooze":
152 case "thread.auto-settle.set":
153 case "thread.pin":
154 case "thread.unpin":
155 case "thread.pin.reorder":
156 case "thread.active.reorder":
157 case "thread.tag.set":
158 case "thread.visit":
159 case "thread.mark-unread":
160 case "thread.metadata.update":
161 case "thread.pull-request.link":
162 case "thread.pull-request.unlink":
163 case "thread.pull-request-link.sync":
164 case "thread.issue.link":
165 case "thread.issue.unlink":
166 case "thread.pull-request.watch":
167 case "thread.pull-request.sync":
168 case "thread.title.regeneration.complete":
169 case "thread.runtime-mode.set":
170 case "thread.interaction-mode.set":
171 case "thread.model-selection.set":
172 case "provider-session.detach":
173 case "message.dispatch":
174 case "notification.delivery.accept":
175 case "prepared-run.release":
176 case "prepared-run.progress":
177 case "prepared-run.fail":
178 case "prepared-run.retry":
179 case "subagent.stop":
180 case "run.interrupt":
181 case "queued-message.promote-to-steer":
182 case "queued-run.start":
183 case "queue.resume":
184 case "queued-run.reorder":
185 case "queued-run.cancel":
186 case "queued-run.edit":
187 case "runtime-request.respond":
188 case "thread.user-input.dismiss":
189 case "checkpoint.rollback":
190 case "provider.switch":
191 return [command.threadId];
192 case "delegated_task.request":
193 case "delegated_task.wake-policy":
194 case "delegated_task.completion-delivery.acknowledge":
195 case "delegated_task.completion-delivery.dispose":
196 case "thread.created.record":
197 return [command.parentThreadId];
198 case "thread.fork":
199 case "thread.merge_back":
200 return [command.sourceThreadId, command.targetThreadId];
201 }
202 }
203
204 function scenarioSteps(
205 scenario: OrchestratorV2Scenario,
206 ): ReadonlyArray<OrchestratorV2ScenarioStep> {
207 return (
208 scenario.steps ??
209 scenario.commands.map((command) => ({
210 type: "dispatch" as const,
211 command,
212 await: true,
213 }))
214 );
215 }
216
217 function scenarioCommands(scenario: OrchestratorV2Scenario): ReadonlyArray<OrchestrationV2Command> {
218 return scenarioSteps(scenario).flatMap((step) =>
219 step.type === "dispatch" ? [step.command] : [],
220 );
221 }
222
223 const findPendingRuntimeRequest = (projection: OrchestrationV2ThreadProjection) =>
224 projection.runtimeRequests.find((request) => request.status === "pending");
225
226 const hasActiveRun = (projection: OrchestrationV2ThreadProjection) =>
227 projection.runs.some((run) =>
228 ["preparing", "queued", "starting", "running", "waiting"].includes(run.status),
229 );
230
231 const SCENARIO_WAIT_ATTEMPTS = 10_000;
232 // Iteration budgets burn through idle event-loop turns during async I/O; exhaust both iterations and wall-clock deadline.
233 const SCENARIO_WAIT_DEADLINE_MS = 60_000;
234 const scenarioWaitDeadline = () => performance.now() + SCENARIO_WAIT_DEADLINE_MS;
235 const scenarioWaitExhausted = (attemptsRemaining: number, deadlineAt: number) =>
236 attemptsRemaining <= 0 && performance.now() >= deadlineAt;
237
238 const yieldToRuntime = Effect.yieldNow.pipe(
239 Effect.andThen(
240 Effect.promise(
241 () =>
242 new Promise<void>((resolve) => {
243 setImmediate(resolve);
244 }),
245 ),
246 ),
247 );
248
249 function collectProjectionThreadIds(scenario: OrchestratorV2Scenario): ReadonlyArray<ThreadId> {
250 if (scenario.projectionThreadIds) {
251 return scenario.projectionThreadIds;
252 }
253
254 const ids = new Set<ThreadId>();
255 for (const command of scenarioCommands(scenario)) {
256 for (const threadId of commandThreadIds(command)) {
257 ids.add(threadId);
258 }
259 }
260 return Array.from(ids);
261 }
262
263 export function runOrchestratorV2Scenario(
264 scenario: OrchestratorV2Scenario,
265 options: {
266 readonly replayGate?: ProviderReplayGate;
267 readonly afterSteps?: Effect.Effect<void>;
268 } = {},
269 ): Effect.Effect<
270 OrchestratorV2ScenarioResult,
271 Orchestrator.OrchestratorV2Error | OrchestratorV2ScenarioStepError,
272 Orchestrator.OrchestratorV2
273 > {
274 return Effect.scoped(
275 Effect.gen(function* () {
276 const orchestrator = yield* Orchestrator.OrchestratorV2;
277 const storedEventGroups: Array<ReadonlyArray<OrchestrationV2StoredEvent>> = [];
278 const observedStoredEvents = yield* Ref.make<Array<OrchestrationV2StoredEvent>>([]);
279 let releaseWatch:
280 | { readonly threadId: ThreadId; readonly released: Deferred.Deferred<void> }
281 | undefined;
282 yield* orchestrator.streamStoredEvents.pipe(
283 Stream.runForEach((stored) =>
284 Ref.update(observedStoredEvents, (existing) => [...existing, stored]).pipe(
285 Effect.andThen(
286 releaseWatch !== undefined &&
287 stored.event.type === "provider-session.updated" &&
288 stored.event.threadId === releaseWatch.threadId &&
289 stored.event.payload.status === "stopped"
290 ? Deferred.succeed(releaseWatch.released, undefined)
291 : Effect.void,
292 ),
293 ),
294 ),
295 Effect.forkScoped,
296 );
297 const backgroundDispatches = new Map<
298 string,
299 Fiber.Fiber<ReadonlyArray<OrchestrationV2StoredEvent>, Orchestrator.OrchestratorV2Error>
300 >();
301 const capturedShellSnapshots = new Map<string, OrchestrationV2ThreadShellSnapshot>();
302 let anonymousBackgroundDispatchIndex = 0;
303
304 const awaitDispatch = Effect.fn("awaitDispatch")(function* (key: string) {
305 const fiber = backgroundDispatches.get(key);
306 if (!fiber) {
307 return yield* new OrchestratorV2ScenarioStepError({
308 scenario: scenario.name,
309 step: `await:${key}`,
310 });
311 }
312 const events = yield* Fiber.join(fiber);
313 backgroundDispatches.delete(key);
314 storedEventGroups.push(events);
315 });
316
317 const waitForPendingRuntimeRequest = (
318 threadId: ThreadId,
319 attemptsRemaining = SCENARIO_WAIT_ATTEMPTS,
320 deadlineAt = scenarioWaitDeadline(),
321 ): Effect.Effect<
322 OrchestrationV2RuntimeRequest,
323 Orchestrator.OrchestratorV2Error | OrchestratorV2ScenarioStepError,
324 never
325 > =>
326 Effect.gen(function* () {
327 const projection = yield* orchestrator.getThreadProjection(threadId);
328 const request = findPendingRuntimeRequest(projection);
329 if (request !== undefined) {
330 return request;
331 }
332 if (scenarioWaitExhausted(attemptsRemaining, deadlineAt)) {
333 const runState = projection.runs.map((run) => `${run.id}:${run.status}`).join(",");
334 return yield* new OrchestratorV2ScenarioStepError({
335 scenario: scenario.name,
336 step: `respond_to_next_runtime_request:${threadId}:runs=${runState}:providerTurns=${projection.providerTurns.length}`,
337 });
338 }
339 yield* yieldToRuntime;
340 return yield* waitForPendingRuntimeRequest(threadId, attemptsRemaining - 1, deadlineAt);
341 });
342
343 const waitForThreadIdle = (
344 threadId: ThreadId,
345 attemptsRemaining = SCENARIO_WAIT_ATTEMPTS,
346 deadlineAt = scenarioWaitDeadline(),
347 ): Effect.Effect<
348 void,
349 Orchestrator.OrchestratorV2Error | OrchestratorV2ScenarioStepError,
350 never
351 > =>
352 Effect.gen(function* () {
353 const projection = yield* orchestrator.getThreadProjection(threadId);
354 if (!hasActiveRun(projection)) {
355 return;
356 }
357 if (scenarioWaitExhausted(attemptsRemaining, deadlineAt)) {
358 const activeRuns = projection.runs
359 .filter((run) =>
360 ["preparing", "queued", "starting", "running", "waiting"].includes(run.status),
361 )
362 .map((run) => `${run.id}:${run.status}`)
363 .join(",");
364 const pendingRequests = projection.runtimeRequests
365 .filter((request) => request.status === "pending")
366 .map((request) => `${request.id}:${request.kind}`)
367 .join(",");
368 return yield* new OrchestratorV2ScenarioStepError({
369 scenario: scenario.name,
370 step: `await_thread_idle:${threadId}:runs=${activeRuns}:requests=${pendingRequests}`,
371 });
372 }
373 yield* yieldToRuntime;
374 return yield* waitForThreadIdle(threadId, attemptsRemaining - 1, deadlineAt);
375 });
376
377 const waitForNoBackgroundWork = (
378 threadId: ThreadId,
379 attemptsRemaining = SCENARIO_WAIT_ATTEMPTS,
380 deadlineAt = scenarioWaitDeadline(),
381 ): Effect.Effect<
382 void,
383 Orchestrator.OrchestratorV2Error | OrchestratorV2ScenarioStepError,
384 never
385 > =>
386 Effect.gen(function* () {
387 const pending = (yield* orchestrator.getThreadShell(threadId))?.pendingBackgroundTasks;
388 if ((pending?.length ?? 0) === 0) {
389 return;
390 }
391 if (scenarioWaitExhausted(attemptsRemaining, deadlineAt)) {
392 return yield* new OrchestratorV2ScenarioStepError({
393 scenario: scenario.name,
394 step: `await_no_background_work:${threadId}:pending=${pending?.map((task) => task.taskId).join(",")}`,
395 });
396 }
397 yield* yieldToRuntime;
398 return yield* waitForNoBackgroundWork(threadId, attemptsRemaining - 1, deadlineAt);
399 });
400
401 const waitForRunSteerable = (
402 threadId: ThreadId,
403 runId: OrchestrationV2Run["id"],
404 attemptsRemaining = SCENARIO_WAIT_ATTEMPTS,
405 deadlineAt = scenarioWaitDeadline(),
406 ): Effect.Effect<
407 void,
408 Orchestrator.OrchestratorV2Error | OrchestratorV2ScenarioStepError,
409 never
410 > =>
411 Effect.gen(function* () {
412 const projection = yield* orchestrator.getThreadProjection(threadId);
413 const run = projection.runs.find((candidate) => candidate.id === runId);
414 const providerTurn = projection.providerTurns.find(
415 (candidate) =>
416 run?.activeAttemptId !== null &&
417 candidate.runAttemptId === run?.activeAttemptId &&
418 candidate.status === "running",
419 );
420 if (run?.status === "running" && providerTurn !== undefined) {
421 return;
422 }
423 if (scenarioWaitExhausted(attemptsRemaining, deadlineAt)) {
424 return yield* new OrchestratorV2ScenarioStepError({
425 scenario: scenario.name,
426 step: `await_run_steerable:${runId}`,
427 });
428 }
429 yield* yieldToRuntime;
430 return yield* waitForRunSteerable(threadId, runId, attemptsRemaining - 1, deadlineAt);
431 });
432
433 const waitForRunStatus = (
434 threadId: ThreadId,
435 runId: OrchestrationV2Run["id"],
436 status: OrchestrationV2Run["status"],
437 attemptsRemaining = SCENARIO_WAIT_ATTEMPTS,
438 deadlineAt = scenarioWaitDeadline(),
439 ): Effect.Effect<
440 void,
441 Orchestrator.OrchestratorV2Error | OrchestratorV2ScenarioStepError,
442 never
443 > =>
444 Effect.gen(function* () {
445 const projection = yield* orchestrator.getThreadProjection(threadId);
446 const run = projection.runs.find((candidate) => candidate.id === runId);
447 if (run?.status === status) {
448 return;
449 }
450 if (scenarioWaitExhausted(attemptsRemaining, deadlineAt)) {
451 return yield* new OrchestratorV2ScenarioStepError({
452 scenario: scenario.name,
453 step: `await_run_status:${runId}:${status}:actual=${run?.status ?? "missing"}`,
454 });
455 }
456 yield* yieldToRuntime;
457 return yield* waitForRunStatus(
458 threadId,
459 runId,
460 status,
461 attemptsRemaining - 1,
462 deadlineAt,
463 );
464 });
465
466 const waitForRunTurnItem = (
467 threadId: ThreadId,
468 runId: OrchestrationV2Run["id"],
469 itemType: OrchestrationV2TurnItem["type"],
470 itemStatus: OrchestrationV2TurnItem["status"] | undefined,
471 inSubagentThreads: boolean,
472 attemptsRemaining = SCENARIO_WAIT_ATTEMPTS,
473 deadlineAt = scenarioWaitDeadline(),
474 ): Effect.Effect<
475 void,
476 Orchestrator.OrchestratorV2Error | OrchestratorV2ScenarioStepError,
477 never
478 > =>
479 Effect.gen(function* () {
480 const projection = yield* orchestrator.getThreadProjection(threadId);
481 const matches = (item: OrchestrationV2TurnItem) =>
482 item.type === itemType && (itemStatus === undefined || item.status === itemStatus);
483 if (!inSubagentThreads) {
484 if (projection.turnItems.some((item) => item.runId === runId && matches(item))) return;
485 } else {
486 for (const subagent of projection.subagents) {
487 if (subagent.runId !== runId || subagent.childThreadId === null) continue;
488 const child = yield* orchestrator.getThreadProjection(subagent.childThreadId);
489 if (child.turnItems.some(matches)) return;
490 }
491 }
492 if (scenarioWaitExhausted(attemptsRemaining, deadlineAt)) {
493 return yield* new OrchestratorV2ScenarioStepError({
494 scenario: scenario.name,
495 step: `await_run_turn_item:${runId}:${itemType}${itemStatus === undefined ? "" : `:${itemStatus}`}`,
496 });
497 }
498 yield* yieldToRuntime;
499 return yield* waitForRunTurnItem(
500 threadId,
501 runId,
502 itemType,
503 itemStatus,
504 inSubagentThreads,
505 attemptsRemaining - 1,
506 deadlineAt,
507 );
508 });
509
510 const releaseReplayGateAfterWaiting = (
511 label: string,
512 threadId: ThreadId,
513 runId: OrchestrationV2Run["id"],
514 attemptsRemaining = SCENARIO_WAIT_ATTEMPTS,
515 ): Effect.Effect<
516 void,
517 Orchestrator.OrchestratorV2Error | OrchestratorV2ScenarioStepError,
518 never
519 > =>
520 Effect.gen(function* () {
521 const projection = yield* orchestrator.getThreadProjection(threadId);
522 const run = projection.runs.find((candidate) => candidate.id === runId);
523 const providerThread = projection.providerThreads.find(
524 (candidate) => candidate.id === run?.providerThreadId,
525 );
526 const pendingTaskCount = providerThread?.pendingBackgroundTasks?.length ?? 0;
527 const gateReached = options.replayGate?.hasReached(label) ?? false;
528 if (
529 gateReached &&
530 run?.status === "completed" &&
531 providerThread !== undefined &&
532 pendingTaskCount > 0
533 ) {
534 options.replayGate?.release(label);
535 yield* waitForProviderBackgroundTasksCleared(threadId, providerThread.id);
536 return;
537 }
538 if (attemptsRemaining <= 0) {
539 options.replayGate?.release(label);
540 return yield* new OrchestratorV2ScenarioStepError({
541 scenario: scenario.name,
542 step: `release_replay_gate_after_waiting:${label}:reached=${gateReached}:run=${run?.status ?? "missing"}:providerThread=${run?.providerThreadId ?? "missing"}:pending=${pendingTaskCount}`,
543 });
544 }
545 yield* yieldToRuntime;
546 return yield* releaseReplayGateAfterWaiting(
547 label,
548 threadId,
549 runId,
550 attemptsRemaining - 1,
551 );
552 });
553
554 const waitForProviderBackgroundTasksCleared = (
555 threadId: ThreadId,
556 providerThreadId: NonNullable<OrchestrationV2Run["providerThreadId"]>,
557 attemptsRemaining = SCENARIO_WAIT_ATTEMPTS,
558 ): Effect.Effect<
559 void,
560 Orchestrator.OrchestratorV2Error | OrchestratorV2ScenarioStepError,
561 never
562 > =>
563 Effect.gen(function* () {
564 const projection = yield* orchestrator.getThreadProjection(threadId);
565 const providerThread = projection.providerThreads.find(
566 (candidate) => candidate.id === providerThreadId,
567 );
568 const hasPendingTasks = (providerThread?.pendingBackgroundTasks?.length ?? 0) > 0;
569 if (!hasPendingTasks && providerThread?.status === "idle") {
570 return;
571 }
572 if (attemptsRemaining <= 0) {
573 const providerState = projection.providerThreads
574 .map(
575 (candidate) =>
576 `${candidate.id}:${candidate.status}:pending=${candidate.pendingBackgroundTasks?.length ?? 0}`,
577 )
578 .join(",");
579 return yield* new OrchestratorV2ScenarioStepError({
580 scenario: scenario.name,
581 step: `await_provider_background_tasks_cleared:${threadId}:target=${providerThreadId}:providers=${providerState}`,
582 });
583 }
584 yield* yieldToRuntime;
585 return yield* waitForProviderBackgroundTasksCleared(
586 threadId,
587 providerThreadId,
588 attemptsRemaining - 1,
589 );
590 });
591
592 let consumedFinishReceipts = 0;
593 const finishHeldRun = Effect.fn("scenario.finishHeldRun")(function* (
594 step: Extract<OrchestratorV2ScenarioStep, { readonly type: "finish_held_run" }>,
595 ) {
596 const gate = options.replayGate;
597 if (gate === undefined) {
598 return yield* new OrchestratorV2ScenarioStepError({
599 scenario: scenario.name,
600 step: `finish_held_run:${step.runId}:no_replay_gate`,
601 });
602 }
603 while (true) {
604 // Only the latest debounce receipt settles the run; late frames rearm it.
605 const armed = yield* Effect.promise(() =>
606 gate.waitForFinishArmed(consumedFinishReceipts),
607 ).pipe(
608 Effect.timeoutOption(Duration.millis(SCENARIO_WAIT_DEADLINE_MS)),
609 Effect.provideService(Clock.Clock, Clock.Clock.defaultValue()),
610 );
611 if (Option.isNone(armed)) {
612 const projection = yield* orchestrator.getThreadProjection(step.threadId);
613 const run = projection.runs.find((candidate) => candidate.id === step.runId);
614 return yield* new OrchestratorV2ScenarioStepError({
615 scenario: scenario.name,
616 step: `finish_held_run:${step.runId}:${step.status}:actual=${run?.status ?? "missing"}:no_finish_armed`,
617 });
618 }
619 consumedFinishReceipts = gate.finishArmedCount();
620 yield* TestClock.adjust(armed.value);
621 const settled = yield* Effect.raceFirst(
622 waitForRunStatus(step.threadId, step.runId, step.status).pipe(Effect.as(true)),
623 Effect.promise(() => gate.waitForFinishArmed(consumedFinishReceipts)).pipe(
624 Effect.as(false),
625 ),
626 );
627 if (settled) {
628 consumedFinishReceipts = gate.finishArmedCount();
629 return;
630 }
631 }
632 });
633
634 const releaseReplayGate = Effect.fn("scenario.releaseReplayGate")(function* (label: string) {
635 const gate = options.replayGate;
636 const reached =
637 gate === undefined ? false : yield* Effect.promise(() => gate.waitForReached(label));
638 if (!reached) {
639 return yield* new OrchestratorV2ScenarioStepError({
640 scenario: scenario.name,
641 step: `release_replay_gate:${label}:not_configured`,
642 });
643 }
644 gate?.release(label);
645 });
646
647 for (const step of scenarioSteps(scenario)) {
648 switch (step.type) {
649 case "dispatch": {
650 if (step.await ?? true) {
651 const result = yield* orchestrator.dispatch(step.command);
652 storedEventGroups.push(result.storedEvents);
653 break;
654 }
655
656 anonymousBackgroundDispatchIndex += 1;
657 const key = step.key ?? `dispatch:${anonymousBackgroundDispatchIndex}`;
658 backgroundDispatches.set(
659 key,
660 yield* orchestrator.dispatch(step.command).pipe(
661 Effect.map((result) => result.storedEvents),
662 Effect.forkScoped,
663 ),
664 );
665 break;
666 }
667 case "advance_clock": {
668 if (step.awaitProviderSessionRelease === undefined) {
669 yield* TestClock.adjust(step.duration);
670 break;
671 }
672 // The release runs on a woken fiber; a dispatch before it commits cancels it.
673 const released = yield* Deferred.make<void>();
674 releaseWatch = { threadId: step.awaitProviderSessionRelease, released };
675 yield* TestClock.adjust(step.duration);
676 const settled = yield* Deferred.await(released).pipe(
677 Effect.timeoutOption(Duration.millis(SCENARIO_WAIT_DEADLINE_MS)),
678 Effect.provideService(Clock.Clock, Clock.Clock.defaultValue()),
679 );
680 releaseWatch = undefined;
681 if (Option.isNone(settled)) {
682 return yield* new OrchestratorV2ScenarioStepError({
683 scenario: scenario.name,
684 step: `advance_clock:${step.awaitProviderSessionRelease}:no_provider_session_release`,
685 });
686 }
687 break;
688 }
689 case "await":
690 yield* awaitDispatch(step.key);
691 break;
692 case "await_all":
693 for (const key of Array.from(backgroundDispatches.keys())) {
694 yield* awaitDispatch(key);
695 }
696 break;
697 case "await_thread_idle":
698 yield* waitForThreadIdle(step.threadId);
699 break;
700 case "await_no_background_work":
701 yield* waitForNoBackgroundWork(step.threadId);
702 break;
703 case "await_run_steerable":
704 yield* waitForRunSteerable(step.threadId, step.runId);
705 break;
706 case "await_run_status":
707 yield* waitForRunStatus(step.threadId, step.runId, step.status);
708 break;
709 case "finish_held_run":
710 yield* finishHeldRun(step);
711 break;
712 case "await_run_turn_item":
713 yield* waitForRunTurnItem(
714 step.threadId,
715 step.runId,
716 step.itemType,
717 step.itemStatus,
718 step.inSubagentThreads === true,
719 );
720 break;
721 case "release_replay_gate_after_waiting":
722 yield* releaseReplayGateAfterWaiting(step.label, step.threadId, step.runId);
723 break;
724 case "release_replay_gate":
725 yield* releaseReplayGate(step.label);
726 break;
727 case "capture_shell_snapshot":
728 capturedShellSnapshots.set(step.key, yield* orchestrator.getShellSnapshot());
729 break;
730 case "respond_to_next_runtime_request": {
731 const request = yield* waitForPendingRuntimeRequest(step.threadId);
732 if (step.shellSnapshotKeyWhilePending !== undefined) {
733 capturedShellSnapshots.set(
734 step.shellSnapshotKeyWhilePending,
735 yield* orchestrator.getShellSnapshot(),
736 );
737 }
738 const result = yield* orchestrator.dispatch({
739 type: "runtime-request.respond",
740 commandId: step.commandId,
741 threadId: step.threadId,
742 requestId: request.id,
743 ...(step.decision === undefined ? {} : { decision: step.decision }),
744 ...(step.answers === undefined ? {} : { answers: step.answers }),
745 });
746 storedEventGroups.push(result.storedEvents);
747 break;
748 }
749 }
750 }
751
752 for (const key of Array.from(backgroundDispatches.keys())) {
753 yield* awaitDispatch(key);
754 }
755 if (options.afterSteps !== undefined) {
756 yield* options.afterSteps;
757 }
758
759 const shellSnapshot = yield* orchestrator.getShellSnapshot();
760 const projectionThreadIds = new Set(collectProjectionThreadIds(scenario));
761 for (const thread of shellSnapshot.threads) {
762 projectionThreadIds.add(thread.id);
763 }
764 const projections = new Map<ThreadId, OrchestrationV2ThreadProjection>();
765 for (const threadId of projectionThreadIds) {
766 projections.set(threadId, yield* orchestrator.getThreadProjection(threadId));
767 }
768
769 yield* Effect.yieldNow;
770
771 const observedEvents = yield* Ref.get(observedStoredEvents);
772 const storedEvents = (
773 observedEvents.length > 0 ? observedEvents : storedEventGroups.flat()
774 ).toSorted((left, right) => left.sequence - right.sequence);
775 return {
776 storedEvents,
777 domainEvents: storedEvents.map((stored) => stored.event),
778 projections,
779 shellSnapshot,
780 capturedShellSnapshots,
781 };
782 }),
783 ).pipe(
784 // Release held provider frames before shutdown so teardown cannot block on gates.
785 Effect.ensuring(Effect.sync(() => options.replayGate?.releaseAll())),
786 );
787 }