The best way to control Claude Code, Codex and more from any computer or phone

main
ts 1,507 lines 59.9 KB
Raw
1 import { assert } from "@effect/vitest";
2 import {
3 type ChatAttachment,
4 CommandId,
5 isOrchestrationV2WorkActive,
6 MessageId,
7 ProjectId,
8 ThreadId,
9 type ModelSelection,
10 type OrchestrationV2Command,
11 type OrchestrationV2ExecutionNode,
12 type OrchestrationV2RunStatus,
13 type OrchestrationV2ThreadProjection,
14 type OrchestrationV2TurnItem,
15 type OrchestrationV2UserMessageInputIntent,
16 ProviderInstanceId,
17 type ProviderInteractionMode,
18 type ProviderDriverKind,
19 type ProviderReplayTranscript,
20 type ProviderUserInputAnswers,
21 type RuntimeMode,
22 } from "@t3tools/contracts";
23 import type * as Duration from "effect/Duration";
24 import * as Effect from "effect/Effect";
25 import * as Predicate from "effect/Predicate";
26
27 import type {
28 OrchestratorV2ScenarioResult,
29 OrchestratorV2ScenarioStep,
30 } from "../OrchestratorScenario.ts";
31 import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
32 import type { RuntimePolicyV2Override } from "../../RuntimePolicy.ts";
33
34 export const SIMPLE_PROMPT = "Respond with the following text: fixture simple ok";
35 export const MULTI_TURN_FIRST_PROMPT = "Respond with exactly: first fixture turn complete";
36 export const MULTI_TURN_SECOND_PROMPT = "Respond with exactly: second fixture turn complete";
37 export const PROVIDER_THREAD_RESUME_FIRST_PROMPT =
38 "Respond with exactly: provider thread resume fixture first turn complete";
39 export const PROVIDER_THREAD_RESUME_SECOND_PROMPT =
40 "Using the conversation history available in this resumed thread, first repeat the exact final answer you gave in the previous turn. Then on a new line write exactly: provider thread resume fixture second turn complete";
41 export const TOOL_CALL_READ_ONLY_WORKSPACE_ROOT = "/tmp/claude-replay-tool_call_read_only";
42 export const TOOL_CALL_READ_ONLY_PROMPT = `Read ${TOOL_CALL_READ_ONLY_WORKSPACE_ROOT}/package.json and ${TOOL_CALL_READ_ONLY_WORKSPACE_ROOT}/tsconfig.json, then answer exactly: read only tool fixture complete`;
43 export const CLAUDE_LOCAL_BASH_TASK_PROMPT =
44 "Run a local Bash typecheck command, then answer exactly: claude local bash task fixture complete";
45 export const CLAUDE_RESULT_IS_ERROR_PROMPT = "Say hello before the credentials expire.";
46 export const CLAUDE_RESULT_IS_ERROR_FOLLOW_UP = "Try again now that auth is back.";
47 export const TOOL_CALL_WRITE_PROMPT =
48 "Create or overwrite .codex-probe-write-action.txt with exactly this text: codex app-server approval fixture. Use a local shell command or file edit only, then briefly report what happened. Do not read package metadata, use GitHub, use web, or use MCP.";
49 export const MESSAGE_STEERING_INITIAL_PROMPT =
50 "Respond with exactly: steering fixture initial response";
51 export const SUBAGENT_PROMPT =
52 "Spawn 2 subagents, one to read package.json and one to read tsconfig.json";
53 export const SUBAGENT_V2_NESTED_PROMPT =
54 "Spawn one subagent and tell it to spawn its own subagent, which must in turn spawn one more subagent whose only task is to reply with exactly: Hello. Each agent waits for its child and replies with exactly what the child said. Wait for your subagent, then reply with exactly what it said.";
55 export const SUBAGENT_V2_PROMPT =
56 "Spawn one subagent whose only task is to reply with exactly: Hello. Wait for it to finish, then reply with exactly what it said.";
57 export const SUBAGENT_V2_APPROVAL_PROMPT =
58 "Do not run any commands yourself. Spawn one subagent whose only task is to run this exact shell command: printf 'subagent approval fixture' > subagent-approval.txt and then reply with exactly: Written. Wait for it to finish, then reply with exactly what it said.";
59 export const SUBAGENT_V2_NESTED_APPROVAL_PROMPT =
60 "Do not run any commands yourself. Spawn one subagent and tell it not to run any commands itself but to spawn its own subagent, whose only task is to run this exact shell command: printf 'nested approval fixture' > nested-approval.txt and then reply with exactly: Written. Each agent waits for its child and replies with exactly what the child said. Wait for your subagent, then reply with exactly what it said.";
61 export const OPENCODE_SUBAGENT_PROMPT =
62 "Use the task tool exactly once. Delegate to the general subagent with this prompt: Respond exactly CHILD_OK. After the task completes, respond exactly PARENT_OK.";
63 export const OPENCODE_BACKGROUND_WAKE_PROMPT =
64 "Use the task tool once with background set to true to delegate to the general subagent with the prompt: Respond exactly CHILD_OK. As soon as it is launched, reply exactly PARENT_OK and end your turn without waiting for it.";
65 export const SUBAGENT_CONTINUE_PROMPT =
66 "Spawn one subagent and have it reply exactly: initial subagent response";
67 export const SUBAGENT_CONTINUE_PARENT_PROMPT =
68 "Have the same subagent you spawned earlier reply exactly: continued subagent response";
69 export const SUBAGENT_CONTINUE_CHILD_PROMPT = "Reply exactly: continued subagent response";
70 export const OPENCODE2_SIMPLE_PROMPT =
71 "Think carefully step by step about whether 391 is prime, showing your reasoning, then answer in one short sentence.";
72 export const OPENCODE2_TOOL_CALL_PROMPT =
73 "Use the read tool to read hello.txt, then run the shell command `echo TOOL_OK` with the bash tool, then reply DONE.";
74 export const OPENCODE2_INTERRUPT_PROMPT =
75 "Run the shell command `sleep 60 && echo LATE` with the bash tool, then reply DONE.";
76 export const OPENCODE2_PERMISSION_PROMPT =
77 "Run the shell command `echo FIRST` with the bash tool. After it completes, run `echo SECOND` with the bash tool. Then reply with what happened.";
78 export const OPENCODE2_STEER_PROMPT =
79 "Run the shell command `sleep 12 && echo A` with the bash tool, then reply DONE_A.";
80 export const OPENCODE2_STEER_TEXT = "Also mention the word STEERED in your final reply.";
81 export const OPENCODE2_QUEUED_PROMPT = "Reply exactly QUEUED_B.";
82 export const OPENCODE2_CANCELLED_PROMPT = "Reply exactly QUEUED_C.";
83 export const OPENCODE2_REVERT_FIRST_PROMPT =
84 "Create a file named reverted.txt containing the word ALPHA using the write tool, then reply DONE.";
85 export const OPENCODE2_REVERT_SECOND_PROMPT =
86 "Overwrite reverted.txt so it contains the word BETA using the write tool, then reply DONE.";
87 export const OPENCODE2_QUESTION_PROMPT =
88 "Before doing anything, use the question tool to ask me which color I prefer, offering the options red and blue. After I answer, reply with only the chosen color.";
89 export const OPENCODE2_SUBAGENT_PROMPT =
90 "Use the subagent tool to delegate to the explore subagent with the prompt: 'List the files in the current directory and report their names.' Wait for it, then summarize its answer in one line.";
91 export const OPENCODE2_NESTED_BACKGROUND_PROMPT =
92 "Use the subagent tool (foreground, do not set background) to delegate to the general subagent with this exact prompt: 'Use the subagent tool with background set to true to delegate to the general subagent with the prompt: Run the shell command `sleep 25` with the shell tool, then reply exactly GRANDCHILD_OK. As soon as it is launched, reply exactly MIDDLE_OK and end your turn without waiting for it.' Wait for that subagent to return, then reply exactly ROOT_OK.";
93 export const OPENCODE2_BACKGROUND_PROMPT =
94 "Use the subagent tool with background enabled to delegate to the general subagent with the prompt: 'Run the shell command `sleep 20` with the bash tool and then reply exactly CHILD_OK.' As soon as it is launched, reply exactly PARENT_OK and end your turn without waiting for it.";
95 export const OPENCODE2_COMPACTION_FIRST_PROMPT = "Remember the codeword PAPAYA. Reply OK.";
96 export const OPENCODE2_COMPACTION_RECALL_PROMPT = "What was the codeword? One word.";
97 export const OPENCODE2_RESTART_PROMPT =
98 "Run the shell command `sleep 25 && echo RESUMED` with the bash tool, then reply with its output.";
99 export const OPENCODE2_RESTART_RECALL_PROMPT =
100 "What did I last ask you to run? Answer in one short sentence.";
101 export const OPENCODE2_COMMAND_PROMPT = "/hello WORLD";
102 export const OPENCODE2_SKILL_PROMPT = "Use $greet to say hi in three words.";
103 export const TURN_INTERRUPT_PROMPT =
104 "Do not answer immediately. First run the local shell command `sleep 30`, then respond with exactly: interrupt fixture should not finish naturally.";
105 export const FX_SUBAGENT_STOP_PROMPT =
106 "Use exactly one subagent to create a file named stopped.txt containing STOP. Do not create the file yourself. Wait for the subagent, then reply DONE.";
107 export const TURN_INTERRUPT_MID_TOOL_PROMPT =
108 "Run this exact local command: `node -e \"console.log('interrupt fixture tool started'); setTimeout(() => {}, 30000)\"`. Do not answer until it completes, then respond exactly: interrupt fixture should not finish naturally.";
109 const COMMAND_OUTPUT_STREAMING_COMMAND =
110 "for i in 1 2 3 4; do printf '\\033[32mstream line %s\\033[0m\\n' $i; sleep 1; done; printf 'progress 50%%\\rprogress 100%%\\n'";
111 export const COMMAND_OUTPUT_STREAMING_PROMPT = `Run this exact shell command in the foreground and wait for it to finish: \`${COMMAND_OUTPUT_STREAMING_COMMAND}\`. Then respond exactly: command output fixture complete`;
112 export const TURN_INTERRUPT_RECOVERY_PROMPT =
113 "Respond with exactly: interrupt recovery fixture complete";
114 export const MESSAGE_STEERING_STEER_PROMPT =
115 "Actually, respond with exactly: steering fixture observed";
116 export const THREAD_ROLLBACK_FIRST_PROMPT =
117 "Respond with exactly: rollback fixture first turn complete";
118 export const THREAD_ROLLBACK_SECOND_PROMPT =
119 "Respond with exactly: rollback fixture second turn complete";
120 export const THREAD_ROLLBACK_AFTER_PROMPT = "Repeat the conversation verbatim.";
121 export const THREAD_FORK_NATIVE_SOURCE_PROMPT =
122 "Respond with the following text: source fork seed ok";
123 export const THREAD_FORK_NATIVE_TARGET_PROMPT = "Respond with the following text: fork native ok";
124 const THREAD_MERGE_BACK_SIBLINGS_SOURCE_MARKER = "merge-sibling-source-3C7K";
125 const THREAD_MERGE_BACK_SIBLINGS_FIRST_MARKER = "merge-sibling-first-6V2J";
126 const THREAD_MERGE_BACK_SIBLINGS_SECOND_MARKER = "merge-sibling-second-9X5B";
127 export const THREAD_MERGE_BACK_SIBLINGS_SOURCE_PROMPT = `Remember the opaque marker ${THREAD_MERGE_BACK_SIBLINGS_SOURCE_MARKER} for later in this conversation. Respond with exactly: merge sibling source stored`;
128 export const THREAD_MERGE_BACK_SIBLINGS_FIRST_FORK_PROMPT = `Remember the fork-local marker ${THREAD_MERGE_BACK_SIBLINGS_FIRST_MARKER}. Respond with exactly: first merge sibling stored`;
129 export const THREAD_MERGE_BACK_SIBLINGS_SECOND_FORK_PROMPT = `Remember the fork-local marker ${THREAD_MERGE_BACK_SIBLINGS_SECOND_MARKER}. Respond with exactly: second merge sibling stored`;
130 export const THREAD_MERGE_BACK_SIBLINGS_FIRST_HANDOFF_PROMPT = [
131 "Context handoff (merge_back / fork_delta_summary):",
132 "Merge-back context from first forked conversation.",
133 "",
134 "Fork delta:",
135 `- User introduced opaque marker ${THREAD_MERGE_BACK_SIBLINGS_FIRST_MARKER}.`,
136 "- Assistant confirmed: first merge sibling stored",
137 "",
138 "User message:",
139 "Retain the first transferred marker for later. Respond with exactly: first merge delta stored",
140 ].join("\n");
141 export const THREAD_MERGE_BACK_SIBLINGS_SECOND_HANDOFF_PROMPT = [
142 "Context handoff (merge_back / fork_delta_summary):",
143 "Merge-back context from second forked conversation.",
144 "",
145 "Fork delta:",
146 `- User introduced opaque marker ${THREAD_MERGE_BACK_SIBLINGS_SECOND_MARKER}.`,
147 "- Assistant confirmed: second merge sibling stored",
148 "",
149 "User message:",
150 "Retain the second transferred marker for later. Respond with exactly: second merge delta stored",
151 ].join("\n");
152 export const THREAD_MERGE_BACK_SIBLINGS_RECALL = [
153 THREAD_MERGE_BACK_SIBLINGS_SOURCE_MARKER,
154 THREAD_MERGE_BACK_SIBLINGS_FIRST_MARKER,
155 THREAD_MERGE_BACK_SIBLINGS_SECOND_MARKER,
156 ].join("|");
157 export const THREAD_MERGE_BACK_SIBLINGS_RECALL_PROMPT =
158 "Return the source marker followed by both transferred fork markers in merge order, separated by single | characters. Respond with only the markers and separators.";
159 export const THREAD_FORK_NATIVE_PRIOR_TURN_ALPHA_PROMPT =
160 "For this fork-boundary fixture, respond with exactly: fork boundary alpha";
161 export const THREAD_FORK_NATIVE_PRIOR_TURN_BETA_PROMPT =
162 "For this fork-boundary fixture, respond with exactly: fork boundary beta";
163 export const THREAD_FORK_NATIVE_PRIOR_TURN_REPEAT_PROMPT =
164 "Repeat the user-visible conversation so far verbatim. Include only user and assistant messages. Do not include hidden system/developer content.";
165 export const TODO_LIST_PROMPT =
166 "Use the update_plan tool to track exactly three steps: inspect package.json, inspect tsconfig.json, report completion. Then read package.json and tsconfig.json, and answer exactly: todo list fixture complete";
167 export const PLAN_QUESTIONS_PROMPT =
168 "Use request_user_input to ask one multiple-choice clarifying question about whether this fixture should prefer strict schemas or UI flexibility. After receiving the answer, respond exactly: plan questions fixture complete";
169 export const PROPOSED_PLAN_PROMPT =
170 "Create a short implementation plan for adding deterministic replay fixtures. Do not ask questions. Present the final plan in a proposed plan block.";
171 export const WEB_SEARCH_PROMPT =
172 "Search the web for FIFA World Cup ticket pricing, then answer exactly: web search fixture complete";
173 export const SKILL_INVOCATION_PROMPT = "$review README.md";
174 export const SKILL_INVOCATION_CURSOR_MESSAGE = "/review README.md";
175
176 export type OrchestratorFixtureInputStep =
177 | {
178 readonly type: "message";
179 readonly text: string;
180 readonly attachments?: ReadonlyArray<ChatAttachment>;
181 }
182 | {
183 /** Forks the thread at a run into a second thread, then sends `text` there. */
184 readonly type: "fork";
185 readonly fromRunIndex: number;
186 readonly text: string;
187 }
188 | {
189 readonly type: "queue_message";
190 readonly text: string;
191 readonly attachments?: ReadonlyArray<ChatAttachment>;
192 readonly createdBy?: Extract<
193 OrchestrationV2Command,
194 { readonly type: "message.dispatch" }
195 >["createdBy"];
196 readonly creationSource?: Extract<
197 OrchestrationV2Command,
198 { readonly type: "message.dispatch" }
199 >["creationSource"];
200 }
201 | {
202 readonly type: "cancel_queued_run";
203 readonly targetRunIndex: number;
204 }
205 | {
206 readonly type: "await_run_status";
207 readonly targetRunIndex: number;
208 readonly status: OrchestrationV2RunStatus;
209 readonly waitForTurnItemType?: OrchestrationV2TurnItem["type"];
210 }
211 | {
212 /** Replay finishes background-held runs through the adapter's debounce receipt. */
213 readonly type: "finish_held_run";
214 readonly targetRunIndex: number;
215 readonly status: OrchestrationV2RunStatus;
216 }
217 | {
218 readonly type: "capture_shell_snapshot";
219 readonly key: string;
220 }
221 | {
222 readonly type: "release_replay_gate";
223 readonly label: string;
224 }
225 | {
226 readonly type: "steer";
227 readonly text: string;
228 readonly attachments?: ReadonlyArray<ChatAttachment>;
229 readonly targetRunIndex: number;
230 }
231 | {
232 readonly type: "restart";
233 readonly text: string;
234 readonly attachments?: ReadonlyArray<ChatAttachment>;
235 readonly targetRunIndex: number;
236 }
237 | {
238 readonly type: "interrupt";
239 readonly targetRunIndex: number;
240 readonly waitForTurnItemType?: OrchestrationV2TurnItem["type"];
241 /** With waitForTurnItemType: wait until that item has this status. */
242 readonly waitForTurnItemStatus?: OrchestrationV2TurnItem["status"];
243 /** With waitForTurnItemType: wait for it in one of the run's subagent threads. */
244 readonly inSubagentThread?: boolean;
245 }
246 | {
247 readonly type: "stop_background_work";
248 readonly targetRunIndex: number;
249 }
250 | {
251 readonly type: "release_replay_gate_after_waiting";
252 readonly label: string;
253 readonly targetRunIndex: number;
254 }
255 | {
256 readonly type: "approve_next_runtime_request";
257 readonly decision?: Extract<
258 OrchestrationV2Command,
259 { readonly type: "runtime-request.respond" }
260 >["decision"];
261 readonly shellSnapshotKeyWhilePending?: string;
262 }
263 | {
264 readonly type: "answer_next_user_input_request";
265 readonly answers: ProviderUserInputAnswers;
266 }
267 | {
268 readonly type: "rollback";
269 readonly checkpointScopeSuffix: string;
270 readonly checkpointSuffix: string;
271 }
272 | {
273 readonly type: "advance_clock";
274 readonly duration: Duration.Input;
275 /** The jump idle-releases the provider session; later steps wait for that release. */
276 readonly releasesProviderSession?: boolean;
277 }
278 | {
279 /** Picks another model of the same provider instance; later messages use it. */
280 readonly type: "set_model";
281 readonly model: string;
282 };
283
284 export interface OrchestratorFixtureInput {
285 readonly interactionMode?: ProviderInteractionMode;
286 readonly runtimeMode?: RuntimeMode;
287 /** Files committed before replay, keyed by relative path; seed the same files when recording so workspace reads match. */
288 readonly workspaceFiles?: Readonly<Record<string, string>>;
289 readonly steps: ReadonlyArray<OrchestratorFixtureInputStep>;
290 }
291
292 export interface ProviderOrchestratorReplayVariant {
293 readonly driver: ProviderDriverKind;
294 readonly transcriptFile: URL;
295 readonly recordedScenario?: string;
296 readonly transcriptEntriesThroughLabel?: string;
297 readonly modelSelection: ModelSelection;
298 readonly runtimePolicyOverride?: RuntimePolicyV2Override;
299 readonly runContinuationWorker?: boolean;
300 readonly expectedAbsentWorkspacePaths?: ReadonlyArray<string>;
301 readonly assertOutput: (
302 result: OrchestratorV2ScenarioResult,
303 transcript: ProviderReplayTranscript,
304 ) => void;
305 }
306
307 export interface OrchestratorReplayFixture {
308 readonly name: string;
309 readonly buildInput: () => OrchestratorFixtureInput;
310 readonly providers: ReadonlyArray<ProviderOrchestratorReplayVariant>;
311 }
312
313 export interface MaterializedOrchestratorFixtureInput {
314 readonly commands: ReadonlyArray<OrchestrationV2Command>;
315 readonly steps: ReadonlyArray<OrchestratorV2ScenarioStep>;
316 readonly projectionThreadIds: ReadonlyArray<ThreadId>;
317 }
318
319 export interface FixtureIds {
320 readonly threadId: ThreadId;
321 readonly projectId: ProjectId;
322 }
323
324 export const CODEX_MODEL_SELECTION = {
325 instanceId: ProviderInstanceId.make("codex"),
326 model: "gpt-6-luna",
327 } satisfies ModelSelection;
328
329 export const CLAUDE_MODEL_SELECTION = {
330 instanceId: ProviderInstanceId.make("claudeAgent"),
331 model: "claude-sonnet-4-6",
332 } satisfies ModelSelection;
333
334 export const CURSOR_MODEL_SELECTION = {
335 instanceId: ProviderInstanceId.make("cursor"),
336 model: "composer-2.5",
337 } satisfies ModelSelection;
338
339 export const MUSE_MODEL_SELECTION = {
340 instanceId: ProviderInstanceId.make("muse"),
341 model: "muse-spark-1.3-contributor",
342 } satisfies ModelSelection;
343
344 export const GROK_MODEL_SELECTION = {
345 instanceId: ProviderInstanceId.make("grok"),
346 model: "grok-build",
347 } satisfies ModelSelection;
348
349 /** fx fixtures run on fx's Default; the replay adapter has no probed model, so it writes none. */
350 export const FX_MODEL_SELECTION = {
351 instanceId: ProviderInstanceId.make("fx"),
352 model: "default",
353 } satisfies ModelSelection;
354
355 /** Kiro's cheapest listed model, so recordings stay cheap and do not depend on "auto" routing. */
356 export const KIRO_MODEL_SELECTION = {
357 instanceId: ProviderInstanceId.make("kiro"),
358 model: "claude-haiku-4.5",
359 } satisfies ModelSelection;
360
361 export const OPENCODE_MODEL_SELECTION = {
362 instanceId: ProviderInstanceId.make("opencode"),
363 model: "openai/gpt-5.4-mini",
364 options: [{ id: "agent", value: "build" }],
365 } satisfies ModelSelection;
366
367 export const OPENCODE2_MODEL_SELECTION = {
368 instanceId: ProviderInstanceId.make("opencode"),
369 model: "opencode/space-bunny-free",
370 options: [{ id: "variant", value: "high" }],
371 } satisfies ModelSelection;
372
373 export const PI_MODEL_SELECTION = {
374 instanceId: ProviderInstanceId.make("pi"),
375 model: "openrouter/deepseek/deepseek-v4-flash",
376 } satisfies ModelSelection;
377
378 export const ACP_REGISTRY_MODEL_SELECTION = {
379 instanceId: ProviderInstanceId.make("acpRegistry"),
380 model: "default",
381 } satisfies ModelSelection;
382
383 export const READ_ONLY_ON_REQUEST_POLICY = {
384 approvalPolicy: "on-request",
385 sandboxPolicy: {
386 type: "readOnly",
387 access: { type: "fullAccess" },
388 networkAccess: false,
389 },
390 } as const satisfies RuntimePolicyV2Override;
391
392 export const READ_ONLY_NEVER_POLICY = {
393 approvalPolicy: "never",
394 sandboxPolicy: {
395 type: "readOnly",
396 access: { type: "fullAccess" },
397 networkAccess: false,
398 },
399 } as const satisfies RuntimePolicyV2Override;
400
401 export const WORKSPACE_NEVER_POLICY = {
402 approvalPolicy: "never",
403 sandboxPolicy: {
404 type: "workspaceWrite",
405 writableRoots: [],
406 readOnlyAccess: { type: "fullAccess" },
407 networkAccess: false,
408 },
409 } as const satisfies RuntimePolicyV2Override;
410
411 export const RESTRICTED_GRANULAR_POLICY = {
412 approvalPolicy: {
413 granular: {
414 mcp_elicitations: true,
415 request_permissions: true,
416 rules: true,
417 sandbox_approval: true,
418 skill_approval: true,
419 },
420 },
421 sandboxPolicy: {
422 type: "readOnly",
423 access: {
424 type: "restricted",
425 includePlatformDefaults: false,
426 readableRoots: [],
427 },
428 networkAccess: false,
429 },
430 } as const satisfies RuntimePolicyV2Override;
431
432 function createThreadCommand(input: {
433 readonly commandId: CommandId;
434 readonly ids: FixtureIds;
435 readonly scenario: string;
436 readonly modelSelection: ModelSelection;
437 readonly interactionMode?: ProviderInteractionMode;
438 readonly runtimeMode?: RuntimeMode;
439 }): OrchestrationV2Command {
440 return {
441 type: "thread.create",
442 createdBy: "user",
443 creationSource: "web",
444 commandId: input.commandId,
445 threadId: input.ids.threadId,
446 projectId: input.ids.projectId,
447 title: `Replay fixture: ${input.scenario}`,
448 modelSelection: input.modelSelection,
449 runtimeMode: input.runtimeMode ?? "full-access",
450 interactionMode: input.interactionMode ?? "default",
451 branch: null,
452 worktreePath: null,
453 };
454 }
455
456 function dispatchMessageCommand(input: {
457 readonly commandId: CommandId;
458 readonly ids: FixtureIds;
459 readonly modelSelection: ModelSelection;
460 readonly messageId: MessageId;
461 readonly text: string;
462 readonly attachments?: ReadonlyArray<ChatAttachment>;
463 readonly createdBy?: Extract<
464 OrchestrationV2Command,
465 { readonly type: "message.dispatch" }
466 >["createdBy"];
467 readonly creationSource?: Extract<
468 OrchestrationV2Command,
469 { readonly type: "message.dispatch" }
470 >["creationSource"];
471 readonly dispatchMode?: Extract<
472 OrchestrationV2Command,
473 { readonly type: "message.dispatch" }
474 >["dispatchMode"];
475 }): OrchestrationV2Command {
476 return {
477 type: "message.dispatch",
478 createdBy: input.createdBy ?? "user",
479 creationSource: input.creationSource ?? "web",
480 commandId: input.commandId,
481 threadId: input.ids.threadId,
482 messageId: input.messageId,
483 text: input.text,
484 attachments: [...(input.attachments ?? [])],
485 modelSelection: input.modelSelection,
486 dispatchMode: input.dispatchMode ?? { type: "start_immediately" },
487 };
488 }
489
490 export function materializeFixtureInput(input: {
491 readonly scenario: string;
492 readonly fixtureInput: OrchestratorFixtureInput;
493 readonly driver: ProviderDriverKind;
494 readonly modelSelection: ModelSelection;
495 }): Effect.Effect<
496 MaterializedOrchestratorFixtureInput,
497 IdAllocator.IdAllocatorV2Error,
498 IdAllocator.IdAllocatorV2
499 > {
500 return Effect.gen(function* () {
501 const idAllocator = yield* IdAllocator.IdAllocatorV2;
502 const projectId = yield* idAllocator.allocate.project({ fixtureName: input.scenario });
503 const threadId = yield* idAllocator.allocate.thread({
504 fixtureName: input.scenario,
505 projectId,
506 });
507 const ids = { threadId, projectId } satisfies FixtureIds;
508 const commands: Array<OrchestrationV2Command> = [];
509 const steps: Array<OrchestratorV2ScenarioStep> = [];
510 let messageIndex = 0;
511 let runIndex = 0;
512 let forkThreadId: ThreadId | undefined;
513 const activeRunDispatchKeys = new Set<string>();
514
515 const runIdFor = (runOrdinal: number) =>
516 idAllocator.derive.run({ threadId: ids.threadId, ordinal: runOrdinal });
517
518 // A `set_model` step changes the selection every later message carries.
519 let modelSelection = input.modelSelection;
520 const pushDispatch = (
521 command: OrchestrationV2Command,
522 options: {
523 readonly await?: boolean;
524 readonly key?: string;
525 readonly advanceClockAfter?: boolean;
526 } = {},
527 ) => {
528 commands.push(command);
529 steps.push({
530 type: "dispatch",
531 command,
532 await: options.await ?? true,
533 ...(options.key === undefined ? {} : { key: options.key }),
534 });
535 if (options.advanceClockAfter ?? true) {
536 steps.push({ type: "advance_clock", duration: "1 millis" });
537 }
538 };
539
540 pushDispatch(
541 createThreadCommand({
542 commandId: yield* idAllocator.allocate.command({
543 fixtureName: input.scenario,
544 commandName: "thread-create",
545 }),
546 ids,
547 scenario: input.scenario,
548 modelSelection,
549 ...(input.fixtureInput.interactionMode === undefined
550 ? {}
551 : { interactionMode: input.fixtureInput.interactionMode }),
552 ...(input.fixtureInput.runtimeMode === undefined
553 ? {}
554 : { runtimeMode: input.fixtureInput.runtimeMode }),
555 }),
556 );
557
558 // Multiple requests keep runs busy between answers; wait for each request, not thread idle.
559 const answersNext = (stepIndex: number) => {
560 const next = input.fixtureInput.steps[stepIndex + 1]?.type;
561 return next === "approve_next_runtime_request" || next === "answer_next_user_input_request";
562 };
563 for (const [stepIndex, step] of input.fixtureInput.steps.entries()) {
564 switch (step.type) {
565 case "message":
566 messageIndex += 1;
567 runIndex += 1;
568 {
569 const nextStep = input.fixtureInput.steps[stepIndex + 1];
570 const shouldRunInBackground =
571 (nextStep !== undefined &&
572 ((nextStep.type === "interrupt" && nextStep.targetRunIndex === runIndex) ||
573 nextStep.type === "queue_message" ||
574 (nextStep.type === "restart" && nextStep.targetRunIndex === runIndex) ||
575 (nextStep.type === "release_replay_gate_after_waiting" &&
576 nextStep.targetRunIndex === runIndex) ||
577 // Continuations start while busy; waiting for idle would deadlock.
578 (nextStep.type === "await_run_status" && nextStep.targetRunIndex > runIndex) ||
579 // Held open until the test clock moves, so it cannot go idle first.
580 nextStep.type === "finish_held_run")) ||
581 nextStep?.type === "approve_next_runtime_request" ||
582 nextStep?.type === "answer_next_user_input_request";
583 const key = `run:${runIndex}`;
584 pushDispatch(
585 dispatchMessageCommand({
586 commandId: yield* idAllocator.allocate.command({
587 fixtureName: input.scenario,
588 commandName: `message-${messageIndex}`,
589 }),
590 ids,
591 modelSelection,
592 messageId: yield* idAllocator.allocate.message({
593 threadId: ids.threadId,
594 ordinal: messageIndex,
595 }),
596 text: step.text,
597 ...(step.attachments === undefined ? {} : { attachments: step.attachments }),
598 }),
599 shouldRunInBackground ? { await: false, key } : undefined,
600 );
601 if (shouldRunInBackground) {
602 activeRunDispatchKeys.add(key);
603 } else if (
604 !(
605 nextStep !== undefined &&
606 (nextStep.type === "steer" || nextStep.type === "restart") &&
607 nextStep.targetRunIndex === runIndex
608 )
609 ) {
610 steps.push({ type: "await_thread_idle", threadId: ids.threadId });
611 }
612 }
613 break;
614 case "queue_message": {
615 messageIndex += 1;
616 runIndex += 1;
617 const nextStep = input.fixtureInput.steps[stepIndex + 1];
618 pushDispatch(
619 dispatchMessageCommand({
620 commandId: yield* idAllocator.allocate.command({
621 fixtureName: input.scenario,
622 commandName: `queue-message-${messageIndex}`,
623 }),
624 ids,
625 modelSelection,
626 messageId: yield* idAllocator.allocate.message({
627 threadId: ids.threadId,
628 ordinal: messageIndex,
629 }),
630 text: step.text,
631 ...(step.createdBy === undefined ? {} : { createdBy: step.createdBy }),
632 ...(step.creationSource === undefined ? {} : { creationSource: step.creationSource }),
633 ...(step.attachments === undefined ? {} : { attachments: step.attachments }),
634 dispatchMode: { type: "queue_after_active" },
635 }),
636 );
637 const shouldSkipQueueBarrier =
638 nextStep?.type === "queue_message" ||
639 (nextStep?.type === "cancel_queued_run" && nextStep.targetRunIndex === runIndex);
640 if (!shouldSkipQueueBarrier) {
641 const queueBarrierKey =
642 Array.from(activeRunDispatchKeys).at(-1) ?? `run:${runIndex - 1}`;
643 activeRunDispatchKeys.delete(queueBarrierKey);
644 steps.push({ type: "await", key: queueBarrierKey });
645 steps.push({ type: "await_thread_idle", threadId: ids.threadId });
646 }
647 break;
648 }
649 case "cancel_queued_run":
650 pushDispatch({
651 type: "queued-run.cancel",
652 commandId: yield* idAllocator.allocate.command({
653 fixtureName: input.scenario,
654 commandName: `cancel-queued-run-${step.targetRunIndex}`,
655 }),
656 threadId: ids.threadId,
657 runId: runIdFor(step.targetRunIndex),
658 });
659 break;
660 case "await_run_status":
661 steps.push({
662 type: "await_run_status",
663 threadId: ids.threadId,
664 runId: runIdFor(step.targetRunIndex),
665 status: step.status,
666 });
667 if (step.waitForTurnItemType !== undefined) {
668 steps.push({
669 type: "await_run_turn_item",
670 threadId: ids.threadId,
671 runId: runIdFor(step.targetRunIndex),
672 itemType: step.waitForTurnItemType,
673 });
674 }
675 break;
676 case "finish_held_run":
677 steps.push({
678 type: "finish_held_run",
679 threadId: ids.threadId,
680 runId: runIdFor(step.targetRunIndex),
681 status: step.status,
682 });
683 break;
684 case "capture_shell_snapshot":
685 steps.push({ type: "capture_shell_snapshot", key: step.key });
686 break;
687 case "release_replay_gate":
688 steps.push({ type: "release_replay_gate", label: step.label });
689 break;
690 case "answer_next_user_input_request":
691 pushDispatch(
692 {
693 type: "runtime-request.respond",
694 commandId: yield* idAllocator.allocate.command({
695 fixtureName: input.scenario,
696 commandName: `answer-user-input-request-${messageIndex}`,
697 }),
698 threadId: ids.threadId,
699 requestId: yield* idAllocator.allocate.runtimeRequest({
700 driver: input.driver,
701 nativeRequestId: `fixture-placeholder-${messageIndex}`,
702 }),
703 answers: step.answers,
704 },
705 { advanceClockAfter: false },
706 );
707 steps[steps.length - 1] = {
708 type: "respond_to_next_runtime_request",
709 threadId: ids.threadId,
710 commandId: commands.at(-1)!.commandId,
711 answers: step.answers,
712 };
713 steps.push({ type: "advance_clock", duration: "1 millis" });
714 if (!answersNext(stepIndex)) {
715 steps.push({ type: "await_thread_idle", threadId: ids.threadId });
716 }
717 break;
718 case "approve_next_runtime_request":
719 pushDispatch(
720 {
721 type: "runtime-request.respond",
722 commandId: yield* idAllocator.allocate.command({
723 fixtureName: input.scenario,
724 commandName: `approve-runtime-request-${messageIndex}`,
725 }),
726 threadId: ids.threadId,
727 requestId: yield* idAllocator.allocate.runtimeRequest({
728 driver: input.driver,
729 nativeRequestId: `fixture-placeholder-${messageIndex}`,
730 }),
731 decision: step.decision ?? "accept",
732 },
733 { advanceClockAfter: false },
734 );
735 steps[steps.length - 1] = {
736 type: "respond_to_next_runtime_request",
737 threadId: ids.threadId,
738 commandId: commands.at(-1)!.commandId,
739 decision: step.decision ?? "accept",
740 ...(step.shellSnapshotKeyWhilePending === undefined
741 ? {}
742 : { shellSnapshotKeyWhilePending: step.shellSnapshotKeyWhilePending }),
743 };
744 steps.push({ type: "advance_clock", duration: "1 millis" });
745 if (!answersNext(stepIndex)) {
746 steps.push({ type: "await_thread_idle", threadId: ids.threadId });
747 }
748 break;
749 case "steer":
750 messageIndex += 1;
751 steps.push({
752 type: "await_run_steerable",
753 threadId: ids.threadId,
754 runId: runIdFor(step.targetRunIndex),
755 });
756 pushDispatch(
757 dispatchMessageCommand({
758 commandId: yield* idAllocator.allocate.command({
759 fixtureName: input.scenario,
760 commandName: `steer-${messageIndex}`,
761 }),
762 ids,
763 modelSelection,
764 messageId: yield* idAllocator.allocate.message({
765 threadId: ids.threadId,
766 ordinal: messageIndex,
767 }),
768 text: step.text,
769 ...(step.attachments === undefined ? {} : { attachments: step.attachments }),
770 dispatchMode: {
771 type: "steer_active",
772 targetRunId: runIdFor(step.targetRunIndex),
773 },
774 }),
775 );
776 {
777 const nextStepType = input.fixtureInput.steps[stepIndex + 1]?.type;
778 if (
779 nextStepType !== "approve_next_runtime_request" &&
780 nextStepType !== "answer_next_user_input_request"
781 ) {
782 if (activeRunDispatchKeys.delete(`run:${step.targetRunIndex}`)) {
783 steps.push({ type: "await", key: `run:${step.targetRunIndex}` });
784 }
785 steps.push({ type: "await_thread_idle", threadId: ids.threadId });
786 }
787 }
788 break;
789 case "restart":
790 messageIndex += 1;
791 steps.push({
792 type: "await_run_steerable",
793 threadId: ids.threadId,
794 runId: runIdFor(step.targetRunIndex),
795 });
796 pushDispatch(
797 dispatchMessageCommand({
798 commandId: yield* idAllocator.allocate.command({
799 fixtureName: input.scenario,
800 commandName: `restart-${messageIndex}`,
801 }),
802 ids,
803 modelSelection,
804 messageId: yield* idAllocator.allocate.message({
805 threadId: ids.threadId,
806 ordinal: messageIndex,
807 }),
808 text: step.text,
809 ...(step.attachments === undefined ? {} : { attachments: step.attachments }),
810 dispatchMode: {
811 type: "restart_active",
812 targetRunId: runIdFor(step.targetRunIndex),
813 },
814 }),
815 );
816 {
817 const nextStepType = input.fixtureInput.steps[stepIndex + 1]?.type;
818 if (
819 nextStepType !== "approve_next_runtime_request" &&
820 nextStepType !== "answer_next_user_input_request"
821 ) {
822 if (activeRunDispatchKeys.delete(`run:${step.targetRunIndex}`)) {
823 steps.push({ type: "await", key: `run:${step.targetRunIndex}` });
824 }
825 steps.push({ type: "await_thread_idle", threadId: ids.threadId });
826 }
827 }
828 break;
829 case "interrupt":
830 steps.push({
831 type: "await_run_steerable",
832 threadId: ids.threadId,
833 runId: runIdFor(step.targetRunIndex),
834 });
835 if (step.waitForTurnItemType !== undefined) {
836 steps.push({
837 type: "await_run_turn_item",
838 threadId: ids.threadId,
839 runId: runIdFor(step.targetRunIndex),
840 itemType: step.waitForTurnItemType,
841 ...(step.waitForTurnItemStatus === undefined
842 ? {}
843 : { itemStatus: step.waitForTurnItemStatus }),
844 ...(step.inSubagentThread === true ? { inSubagentThreads: true } : {}),
845 });
846 }
847 pushDispatch(
848 {
849 type: "run.interrupt",
850 commandId: yield* idAllocator.allocate.command({
851 fixtureName: input.scenario,
852 commandName: `interrupt-${step.targetRunIndex}`,
853 }),
854 threadId: ids.threadId,
855 runId: runIdFor(step.targetRunIndex),
856 },
857 { advanceClockAfter: false },
858 );
859 if (activeRunDispatchKeys.delete(`run:${step.targetRunIndex}`)) {
860 steps.push({ type: "await", key: `run:${step.targetRunIndex}` });
861 }
862 steps.push({ type: "advance_clock", duration: "1 millis" });
863 steps.push({ type: "await_thread_idle", threadId: ids.threadId });
864 break;
865 case "stop_background_work":
866 steps.push({ type: "await_thread_idle", threadId: ids.threadId });
867 steps.push({ type: "capture_shell_snapshot", key: "before-stop" });
868 pushDispatch(
869 {
870 type: "run.interrupt",
871 commandId: yield* idAllocator.allocate.command({
872 fixtureName: input.scenario,
873 commandName: `stop-background-work-${step.targetRunIndex}`,
874 }),
875 threadId: ids.threadId,
876 runId: runIdFor(step.targetRunIndex),
877 holdQueue: true,
878 },
879 { advanceClockAfter: false },
880 );
881 steps.push({ type: "await_no_background_work", threadId: ids.threadId });
882 break;
883 case "release_replay_gate_after_waiting":
884 steps.push({
885 type: "release_replay_gate_after_waiting",
886 label: step.label,
887 threadId: ids.threadId,
888 runId: runIdFor(step.targetRunIndex),
889 });
890 break;
891 case "advance_clock":
892 steps.push({
893 type: "advance_clock",
894 duration: step.duration,
895 ...(step.releasesProviderSession === true
896 ? { awaitProviderSessionRelease: ids.threadId }
897 : {}),
898 });
899 break;
900 case "set_model":
901 modelSelection = { ...input.modelSelection, model: step.model };
902 pushDispatch({
903 type: "thread.model-selection.set",
904 commandId: yield* idAllocator.allocate.command({
905 fixtureName: input.scenario,
906 commandName: `set-model-${step.model}`,
907 }),
908 threadId: ids.threadId,
909 modelSelection,
910 });
911 break;
912 case "fork":
913 {
914 const targetThreadId = yield* idAllocator.allocate.thread({
915 fixtureName: `${input.scenario}-fork`,
916 projectId: ids.projectId,
917 });
918 forkThreadId = targetThreadId;
919 pushDispatch({
920 type: "thread.fork",
921 createdBy: "user",
922 creationSource: "web",
923 commandId: yield* idAllocator.allocate.command({
924 fixtureName: input.scenario,
925 commandName: "thread-fork",
926 }),
927 sourceThreadId: ids.threadId,
928 targetThreadId,
929 sourcePoint: { type: "run", runId: runIdFor(step.fromRunIndex) },
930 title: "Forked thread",
931 });
932 pushDispatch(
933 dispatchMessageCommand({
934 commandId: yield* idAllocator.allocate.command({
935 fixtureName: input.scenario,
936 commandName: "fork-message",
937 }),
938 ids: { ...ids, threadId: targetThreadId },
939 modelSelection,
940 messageId: yield* idAllocator.allocate.message({
941 threadId: targetThreadId,
942 ordinal: 1,
943 }),
944 text: step.text,
945 }),
946 );
947 steps.push({ type: "await_thread_idle", threadId: targetThreadId });
948 }
949 break;
950 case "rollback":
951 {
952 const scopeId = yield* idAllocator.allocate.checkpointScope({
953 threadId: ids.threadId,
954 name: step.checkpointScopeSuffix,
955 });
956 pushDispatch({
957 type: "checkpoint.rollback",
958 restoreFiles: false,
959 commandId: yield* idAllocator.allocate.command({
960 fixtureName: input.scenario,
961 commandName: `rollback-${step.checkpointSuffix}`,
962 }),
963 threadId: ids.threadId,
964 scopeId,
965 checkpointId: yield* idAllocator.allocate.checkpoint({
966 checkpointScopeId: scopeId,
967 name: step.checkpointSuffix,
968 }),
969 });
970 }
971 break;
972 }
973 }
974
975 if (activeRunDispatchKeys.size > 0) {
976 steps.push({ type: "await_all" });
977 steps.push({ type: "await_thread_idle", threadId: ids.threadId });
978 }
979
980 return {
981 commands,
982 steps,
983 projectionThreadIds:
984 forkThreadId === undefined ? [ids.threadId] : [ids.threadId, forkThreadId],
985 };
986 });
987 }
988
989 /** Reads `key` from a recorded frame or payload of unknown shape. */
990 export function field(value: unknown, key: string): unknown {
991 return Predicate.hasProperty(value, key) ? value[key] : undefined;
992 }
993
994 export function projectionFor(
995 result: OrchestratorV2ScenarioResult,
996 scenario: string,
997 ): OrchestrationV2ThreadProjection {
998 const projections = [...result.projections.values()].filter(
999 (projection) => projection.thread.lineage.parentThreadId === null,
1000 );
1001
1002 assert.equal(projections.length, 1, `expected one root projection for ${scenario}`);
1003 const projection = projections[0];
1004 assert.isDefined(projection, `missing projection for ${scenario}`);
1005 return projection;
1006 }
1007
1008 export function assertBaseProjection(input: {
1009 readonly result: OrchestratorV2ScenarioResult;
1010 readonly transcript: ProviderReplayTranscript;
1011 readonly runCount: number;
1012 readonly providerTurnCountAtLeast?: number;
1013 readonly runStatuses?: ReadonlyArray<OrchestrationV2RunStatus>;
1014 }) {
1015 const projection = projectionFor(input.result, input.transcript.scenario);
1016
1017 assert.equal(
1018 projection.thread.providerInstanceId,
1019 ProviderInstanceId.make(input.transcript.provider),
1020 );
1021 assert.lengthOf(projection.runs, input.runCount);
1022 assert.isAtLeast(projection.providerThreads.length, 1);
1023 assert.isAtLeast(
1024 projection.providerTurns.length,
1025 input.providerTurnCountAtLeast ?? input.runCount,
1026 `expected provider turns; runs=${projection.runs.map((run) => `${run.id}:${run.status}`).join(",")}; sessions=${projection.providerSessions.map((session) => `${session.id}:${session.status}`).join(",")}; items=${projection.turnItems.map((item) => (item.type === "error" ? `${item.type}:${item.failure.message}` : item.type)).join(",")}`,
1027 );
1028 assert.isAtLeast(input.result.domainEvents.length, 1);
1029 assert.deepEqual(
1030 input.result.storedEvents.map((stored) => stored.sequence),
1031 input.result.storedEvents.map((_, index) => index + 1),
1032 );
1033 assert.deepEqual(
1034 input.result.storedEvents.map((stored) => stored.event.id),
1035 input.result.domainEvents.map((event) => event.id),
1036 );
1037
1038 if (input.runStatuses) {
1039 assert.deepEqual(
1040 projection.runs.map((run) => run.status),
1041 input.runStatuses,
1042 );
1043 }
1044 }
1045
1046 export function assertRunOrdinals(
1047 projection: OrchestrationV2ThreadProjection,
1048 expectedOrdinals: ReadonlyArray<number>,
1049 ) {
1050 assert.deepEqual(
1051 projection.runs.map((run) => run.ordinal),
1052 expectedOrdinals,
1053 );
1054 }
1055
1056 function assertRunsHaveRootNodes(projection: OrchestrationV2ThreadProjection) {
1057 for (const run of projection.runs) {
1058 assert.isNotNull(run.rootNodeId, `run ${run.id} must have a root node`);
1059 assert.isTrue(
1060 projection.nodes.some((node) => node.id === run.rootNodeId && node.kind === "root_turn"),
1061 `run ${run.id} root node must exist`,
1062 );
1063 }
1064 }
1065
1066 function assertRootNodesCountForRuns(projection: OrchestrationV2ThreadProjection) {
1067 const rootNodes = projection.nodes.filter((node) => node.kind === "root_turn");
1068 assert.isAtLeast(rootNodes.length, projection.runs.length);
1069 for (const node of rootNodes) {
1070 assert.equal(node.countsForRun, true, `root node ${node.id} must count for its app run`);
1071 }
1072 }
1073
1074 function assertProviderTurnsReferenceNodes(projection: OrchestrationV2ThreadProjection) {
1075 for (const providerTurn of projection.providerTurns) {
1076 assert.isTrue(
1077 projection.nodes.some((node) => node.id === providerTurn.nodeId),
1078 `provider turn ${providerTurn.id} must reference an execution node`,
1079 );
1080 assert.isTrue(
1081 projection.providerThreads.some((thread) => thread.id === providerTurn.providerThreadId),
1082 `provider turn ${providerTurn.id} must reference a provider thread`,
1083 );
1084 }
1085 }
1086
1087 function assertTurnItemsAreOrdered(projection: OrchestrationV2ThreadProjection) {
1088 const ordinals = projection.turnItems.map((item) => item.ordinal);
1089 assert.deepEqual(
1090 ordinals,
1091 [...ordinals].toSorted((left, right) => left - right),
1092 );
1093 }
1094
1095 function assertTurnItemsReferenceProjection(projection: OrchestrationV2ThreadProjection) {
1096 for (const item of projection.turnItems) {
1097 if (item.runId !== null) {
1098 assert.isTrue(
1099 projection.runs.some((run) => run.id === item.runId),
1100 `turn item ${item.id} must reference an existing run`,
1101 );
1102 }
1103 if (item.nodeId !== null) {
1104 assert.isTrue(
1105 projection.nodes.some((node) => node.id === item.nodeId),
1106 `turn item ${item.id} must reference an existing node`,
1107 );
1108 }
1109 if (item.providerTurnId !== null) {
1110 assert.isTrue(
1111 projection.providerTurns.some((turn) => turn.id === item.providerTurnId),
1112 `turn item ${item.id} must reference an existing provider turn`,
1113 );
1114 }
1115 }
1116 }
1117
1118 export function assertVisibleTurnItemsMirrorLocalTurnItems(
1119 projection: OrchestrationV2ThreadProjection,
1120 ) {
1121 assert.lengthOf(
1122 projection.visibleTurnItems,
1123 projection.turnItems.length,
1124 "non-fork visible turn items must mirror local canonical turn items",
1125 );
1126
1127 for (const [index, item] of projection.turnItems.entries()) {
1128 const visibleItem = projection.visibleTurnItems[index];
1129 assert.isDefined(visibleItem, `missing visible turn item at position ${index}`);
1130 assert.equal(visibleItem.position, index);
1131 assert.equal(visibleItem.visibility, "local");
1132 assert.equal(visibleItem.sourceThreadId, item.threadId);
1133 assert.equal(visibleItem.sourceItemId, item.id);
1134 assert.deepEqual(visibleItem.item, item);
1135 }
1136 }
1137
1138 function assertMessagesReferenceProjection(projection: OrchestrationV2ThreadProjection) {
1139 for (const message of projection.messages) {
1140 if (message.runId !== null) {
1141 assert.isTrue(
1142 projection.runs.some((run) => run.id === message.runId),
1143 `message ${message.id} must reference an existing run`,
1144 );
1145 }
1146 if (message.nodeId !== null) {
1147 assert.isTrue(
1148 projection.nodes.some((node) => node.id === message.nodeId),
1149 `message ${message.id} must reference an existing node`,
1150 );
1151 }
1152 }
1153 }
1154
1155 function assertRuntimeRequestsReferenceProjection(projection: OrchestrationV2ThreadProjection) {
1156 for (const request of projection.runtimeRequests) {
1157 const requestNode = projection.nodes.find((node) => node.id === request.nodeId);
1158 assert.isTrue(
1159 requestNode !== undefined,
1160 `runtime request ${request.id} must reference an existing node`,
1161 );
1162 if (
1163 requestNode !== undefined &&
1164 (request.kind === "command" || request.kind === "file-read" || request.kind === "file-change")
1165 ) {
1166 assert.equal(
1167 requestNode.kind,
1168 "approval_request",
1169 `runtime request ${request.id} must reference an approval request node`,
1170 );
1171 }
1172 if (request.providerTurnId !== null) {
1173 assert.isTrue(
1174 projection.providerTurns.some((turn) => turn.id === request.providerTurnId),
1175 `runtime request ${request.id} must reference an existing provider turn`,
1176 );
1177 }
1178 }
1179 }
1180
1181 export function assertSemanticProjectionIntegrity(projection: OrchestrationV2ThreadProjection) {
1182 assertRunsHaveRootNodes(projection);
1183 assertRootNodesCountForRuns(projection);
1184 assertProviderTurnsReferenceNodes(projection);
1185 assertTurnItemsAreOrdered(projection);
1186 assertTurnItemsReferenceProjection(projection);
1187 assertMessagesReferenceProjection(projection);
1188 assertRuntimeRequestsReferenceProjection(projection);
1189 }
1190
1191 export function assertRunProviderTurnCardinality(input: {
1192 readonly projection: OrchestrationV2ThreadProjection;
1193 readonly rootRunCount: number;
1194 readonly providerTurnCountAtLeast?: number;
1195 }) {
1196 assert.lengthOf(input.projection.runs, input.rootRunCount);
1197 assert.isAtLeast(
1198 input.projection.providerTurns.length,
1199 input.providerTurnCountAtLeast ?? input.rootRunCount,
1200 );
1201 }
1202
1203 export function assertNoExtraAppRunsForProviderChildren(input: {
1204 readonly projection: OrchestrationV2ThreadProjection;
1205 readonly expectedAppRuns: number;
1206 }) {
1207 assert.lengthOf(
1208 input.projection.runs,
1209 input.expectedAppRuns,
1210 "provider child activity must not create additional app runs",
1211 );
1212 }
1213
1214 export function assertProviderNativeSubagentRootTurns(result: OrchestratorV2ScenarioResult) {
1215 const activity = (statuses: ReadonlyArray<OrchestrationV2ExecutionNode["status"]>) =>
1216 statuses
1217 .map((status) => (isOrchestrationV2WorkActive(status) ? "active" : status))
1218 .filter((status, index, all) => status !== all[index - 1]);
1219 for (const projection of result.projections.values()) {
1220 for (const subagent of projection.subagents) {
1221 if (subagent.origin !== "provider_native" || subagent.childThreadId === null) continue;
1222 const childThreadId = subagent.childThreadId;
1223 const child = result.projections.get(childThreadId);
1224 assert.isDefined(child, `missing child thread for subagent ${subagent.id}`);
1225 assert.equal(child.thread.creationSource, "provider");
1226 assert.deepEqual(child.thread.forkedFrom, { type: "node", nodeId: subagent.id });
1227 assert.lengthOf(child.runs, 0);
1228 const roots = child.nodes.filter((node) => node.kind === "root_turn");
1229 assert.isNotEmpty(roots, `child ${childThreadId} must have a root turn`);
1230 for (const root of roots) assert.isNull(root.runId);
1231
1232 const runByChildUpdate = new Map<string, string | undefined>();
1233 for (const event of result.domainEvents) {
1234 if (event.threadId !== childThreadId) continue;
1235 const update = `${event.type}:${JSON.stringify(event.payload)}`;
1236 const storedBy = runByChildUpdate.get(update);
1237 if (runByChildUpdate.has(update) && storedBy !== event.runId) {
1238 assert.fail(
1239 `child ${childThreadId} stored ${event.type} in ${storedBy} and ${event.runId}`,
1240 );
1241 }
1242 runByChildUpdate.set(update, event.runId);
1243 }
1244
1245 const rootEvents = result.domainEvents.flatMap((event, index) =>
1246 event.type === "node.updated" &&
1247 event.payload.threadId === childThreadId &&
1248 event.payload.kind === "root_turn"
1249 ? [{ index, status: event.payload.status }]
1250 : [],
1251 );
1252 const firstItemIndex = result.domainEvents.findIndex(
1253 (event) =>
1254 event.type === "turn-item.updated" &&
1255 event.payload.threadId === childThreadId &&
1256 event.payload.type !== "user_message",
1257 );
1258 assert.equal(rootEvents[0]?.status, "running");
1259 if (firstItemIndex !== -1) {
1260 assert.isBelow(
1261 rootEvents[0]?.index ?? Infinity,
1262 firstItemIndex,
1263 `child ${childThreadId} must be working before its first item`,
1264 );
1265 }
1266 const subagentStatuses = result.domainEvents.flatMap((event) =>
1267 event.type === "subagent.updated" && event.payload.id === subagent.id
1268 ? [event.payload.status]
1269 : [],
1270 );
1271 const rootActivity = activity(rootEvents.map((event) => event.status));
1272 const subagentActivity = activity(subagentStatuses);
1273 const wokenAfterEnd =
1274 child.subagents.length > 0 && rootActivity.length > subagentActivity.length;
1275 assert.deepEqual(
1276 wokenAfterEnd ? rootActivity.slice(0, subagentActivity.length) : rootActivity,
1277 subagentActivity,
1278 `child ${childThreadId} root turn must follow subagent ${subagent.id}`,
1279 );
1280 if (wokenAfterEnd) {
1281 assert.notEqual(rootActivity.at(-1), "active", `child ${childThreadId} must end its turns`);
1282 const childSubagentIds = new Set(child.subagents.map((nested) => nested.id));
1283 const nestedEndIndexes = result.domainEvents.flatMap((event, index) =>
1284 event.type === "subagent.updated" &&
1285 childSubagentIds.has(event.payload.id) &&
1286 !isOrchestrationV2WorkActive(event.payload.status)
1287 ? [index]
1288 : [],
1289 );
1290 const subagentEndIndex = result.domainEvents.findLastIndex(
1291 (event) =>
1292 event.type === "subagent.updated" &&
1293 event.payload.id === subagent.id &&
1294 !isOrchestrationV2WorkActive(event.payload.status),
1295 );
1296 const wakeStarts = rootEvents.filter(
1297 (event, position) =>
1298 event.index > subagentEndIndex &&
1299 isOrchestrationV2WorkActive(event.status) &&
1300 !isOrchestrationV2WorkActive(rootEvents[position - 1]?.status ?? "completed"),
1301 );
1302 assert.isNotEmpty(wakeStarts, `child ${childThreadId} woke without a new turn`);
1303 assert.isAtMost(
1304 wakeStarts.length,
1305 nestedEndIndexes.length,
1306 `child ${childThreadId} woke more often than its subagents ended`,
1307 );
1308 for (const wake of wakeStarts) {
1309 assert.isTrue(
1310 nestedEndIndexes.some((endIndex) => endIndex < wake.index),
1311 `child ${childThreadId} woke before any of its subagents ended`,
1312 );
1313 }
1314 }
1315 }
1316 }
1317 }
1318
1319 export function assertExecutionNodeKinds(
1320 projection: OrchestrationV2ThreadProjection,
1321 expectedKinds: ReadonlyArray<OrchestrationV2ExecutionNode["kind"]>,
1322 ) {
1323 const kinds = projection.nodes.map((node) => node.kind);
1324 for (const expectedKind of expectedKinds) {
1325 assert.include(kinds, expectedKind);
1326 }
1327 }
1328
1329 export function assertTurnItemTypes(
1330 projection: OrchestrationV2ThreadProjection,
1331 expectedTypes: ReadonlyArray<OrchestrationV2TurnItem["type"]>,
1332 ) {
1333 const actualTypes = projection.turnItems.map((item) => item.type);
1334 for (const expectedType of expectedTypes) {
1335 assert.include(actualTypes, expectedType);
1336 }
1337 }
1338
1339 export function assertTurnItemTypeSequence(
1340 projection: OrchestrationV2ThreadProjection,
1341 expectedTypes: ReadonlyArray<OrchestrationV2TurnItem["type"]>,
1342 ) {
1343 assert.deepEqual(
1344 projection.turnItems.map((item) => item.type),
1345 expectedTypes,
1346 );
1347 }
1348
1349 export function assertVisibleTurnItemTypeSequence(
1350 projection: OrchestrationV2ThreadProjection,
1351 expectedTypes: ReadonlyArray<OrchestrationV2TurnItem["type"]>,
1352 ) {
1353 assert.deepEqual(
1354 projection.visibleTurnItems.map((row) => row.item.type),
1355 expectedTypes,
1356 );
1357 }
1358
1359 export function assertAssistantTextIncludes(
1360 projection: OrchestrationV2ThreadProjection,
1361 expectedText: string,
1362 ) {
1363 assert.isTrue(
1364 projection.turnItems.some(
1365 (item) => item.type === "assistant_message" && item.text.includes(expectedText),
1366 ),
1367 `expected assistant output to include ${JSON.stringify(expectedText)}`,
1368 );
1369 }
1370
1371 export function assertRuntimeRequestCounts(
1372 projection: OrchestrationV2ThreadProjection,
1373 expected: { readonly total: number; readonly resolved?: number },
1374 ) {
1375 assert.lengthOf(projection.runtimeRequests, expected.total);
1376 if (expected.resolved !== undefined) {
1377 assert.equal(
1378 projection.runtimeRequests.filter((request) => request.status === "resolved").length,
1379 expected.resolved,
1380 );
1381 }
1382 }
1383
1384 function countReplayLabelsWithPrefix(transcript: ProviderReplayTranscript, prefix: string): number {
1385 return transcript.entries.filter(
1386 (entry) => entry.type !== "runtime_exit" && (entry.label?.startsWith(prefix) ?? false),
1387 ).length;
1388 }
1389
1390 export function assertReplayLabelPrefixCount(
1391 transcript: ProviderReplayTranscript,
1392 prefix: string,
1393 expected: number,
1394 ) {
1395 assert.equal(countReplayLabelsWithPrefix(transcript, prefix), expected);
1396 }
1397
1398 export function assertRuntimeRequestKinds(
1399 projection: OrchestrationV2ThreadProjection,
1400 expectedKinds: ReadonlyArray<string>,
1401 ) {
1402 assert.deepEqual(
1403 projection.runtimeRequests.map((request) => request.kind),
1404 expectedKinds,
1405 );
1406 }
1407
1408 export function assertAllRuntimeRequestsResolved(projection: OrchestrationV2ThreadProjection) {
1409 assert.deepEqual(
1410 projection.runtimeRequests.map((request) => request.status),
1411 projection.runtimeRequests.map(() => "resolved"),
1412 );
1413 }
1414
1415 export function assertConversationMessageRoles(
1416 projection: OrchestrationV2ThreadProjection,
1417 expectedRoles: ReadonlyArray<string>,
1418 ) {
1419 assert.deepEqual(
1420 projection.messages.map((message) => message.role),
1421 expectedRoles,
1422 );
1423 }
1424
1425 export function assertNoAcpClientFileOrTerminalRequests(transcript: ProviderReplayTranscript) {
1426 const frames = transcript.entries.flatMap((entry) =>
1427 entry.type === "runtime_exit"
1428 ? []
1429 : [entry.frame as { method?: unknown; params?: { clientCapabilities?: unknown } }],
1430 );
1431 assert.deepInclude(
1432 frames.find((frame) => frame.method === "initialize")?.params?.clientCapabilities ?? {},
1433 { fs: { readTextFile: false, writeTextFile: false }, terminal: false },
1434 "T3 must not advertise client fs or terminals",
1435 );
1436 assert.deepEqual(
1437 frames.flatMap((frame) =>
1438 typeof frame.method === "string" && /^(fs|terminal)\//u.test(frame.method)
1439 ? [frame.method]
1440 : [],
1441 ),
1442 [],
1443 "the agent must not route file or terminal work through T3",
1444 );
1445 }
1446
1447 export function assertUserMessagesInclude(
1448 projection: OrchestrationV2ThreadProjection,
1449 expectedTexts: ReadonlyArray<string>,
1450 ) {
1451 for (const expectedText of expectedTexts) {
1452 assert.isTrue(
1453 projection.turnItems.some(
1454 (item) => item.type === "user_message" && item.text.includes(expectedText),
1455 ),
1456 `expected user input to include ${JSON.stringify(expectedText)}`,
1457 );
1458 }
1459 }
1460
1461 export function assertVisibleUserMessagesInclude(
1462 projection: OrchestrationV2ThreadProjection,
1463 expectedTexts: ReadonlyArray<string>,
1464 ) {
1465 for (const expectedText of expectedTexts) {
1466 assert.isTrue(
1467 projection.visibleTurnItems.some(
1468 (row) => row.item.type === "user_message" && row.item.text.includes(expectedText),
1469 ),
1470 `expected visible user input to include ${JSON.stringify(expectedText)}`,
1471 );
1472 }
1473 }
1474
1475 export function assertVisibleUserMessagesExclude(
1476 projection: OrchestrationV2ThreadProjection,
1477 rejectedTexts: ReadonlyArray<string>,
1478 ) {
1479 for (const rejectedText of rejectedTexts) {
1480 assert.isFalse(
1481 projection.visibleTurnItems.some(
1482 (row) => row.item.type === "user_message" && row.item.text.includes(rejectedText),
1483 ),
1484 `expected visible user input to exclude ${JSON.stringify(rejectedText)}`,
1485 );
1486 }
1487 }
1488
1489 export function assertUserMessageInputIntents(
1490 projection: OrchestrationV2ThreadProjection,
1491 expectedIntents: ReadonlyArray<OrchestrationV2UserMessageInputIntent>,
1492 ) {
1493 assert.deepEqual(
1494 projection.turnItems
1495 .filter((item) => item.type === "user_message")
1496 .map((item) => item.inputIntent),
1497 expectedIntents,
1498 );
1499 }
1500
1501 export function backgroundNotifications(projection: OrchestrationV2ThreadProjection) {
1502 return projection.turnItems.flatMap((item) =>
1503 item.type === "notification"
1504 ? [{ summary: item.summary, outcome: item.outcome, source: item.source }]
1505 : [],
1506 );
1507 }