@devries-cameron20 / t3code Mirror · read-only / commits / dd8452a065
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 20:45 UTC

test(server): split replay fixtures into per-provider files (#22717)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Theo Browne committed Oct 21, 2026 at 00:18 UTC dd8452a06599732784c8f1725fde5e8695d652f2
11 files changed +817 -707
apps/server/src/orchestration-v2/testkit/AcpRegistryReplayFixtures.integration.test.ts new
+213
@@ -0,0 +1,213 @@
1 +import * as NodeServices from "@effect/platform-node/NodeServices";
2 +import { assert, describe, it } from "@effect/vitest";
3 +import {
4 + ProviderDriverKind,
5 + ProviderInstanceId,
6 + type ProviderReplayTranscript,
7 +} from "@t3tools/contracts";
8 +import * as DateTime from "effect/DateTime";
9 +import * as Effect from "effect/Effect";
10 +import * as Layer from "effect/Layer";
11 +import * as Option from "effect/Option";
12 +import * as Stream from "effect/Stream";
13 +import { TestClock } from "effect/testing";
14 +
15 +import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
16 +import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace";
17 +import { AcpRegistryOrchestratorReplayHarness } from "../Adapters/AcpRegistryAdapterV2.testkit.ts";
18 +import * as EffectWorker from "../EffectWorker.ts";
19 +import * as Orchestrator from "../Orchestrator.ts";
20 +import { provideDeterministicTestRuntime } from "./DeterministicRuntime.ts";
21 +import { materializeFixtureInput, SIMPLE_PROMPT } from "./fixtures/shared.ts";
22 +import {
23 + itRunsReplayFixtures,
24 + readReplayFixtureTranscript,
25 + runReplayFixture,
26 +} from "./ProviderReplayFixtures.ts";
27 +import { layerProviderReplay } from "./ProviderReplayHarness.ts";
28 +import { materializeReplayTranscriptRuntimeInstructions } from "./ReplayRuntimeInstructions.ts";
29 +
30 +describe("ACP registry replay fixtures", () => {
31 + it.effect.each([
32 + { name: "open", firstTurn: false, code: -32602 },
33 + { name: "switch", firstTurn: true, code: -32602 },
34 + { name: "authentication", firstTurn: false, code: -32000 },
35 + ])("preserves the ACP model-write $name failure through orchestration", (testCase) =>
36 + Effect.gen(function* () {
37 + const name = `acp_model_write_${testCase.name}`;
38 + const sessionId = "grok-replay-session-1";
39 + const rejectedModel = "replay-model-b";
40 + const modelSelection = {
41 + instanceId: ProviderInstanceId.make("acpRegistry"),
42 + model: rejectedModel,
43 + };
44 + const recorded = yield* readReplayFixtureTranscript(
45 + new URL("./fixtures/simple/registry_transcript.ndjson", import.meta.url),
46 + );
47 + const sessionIndex = recorded.entries.findIndex(
48 + (entry) => "label" in entry && entry.label === "session.created",
49 + );
50 + assert.isAtLeast(sessionIndex, 0);
51 + const transcript = {
52 + ...recorded,
53 + scenario: name,
54 + entries: [
55 + ...recorded.entries.slice(0, sessionIndex),
56 + {
57 + type: "emit_inbound",
58 + label: "session.created",
59 + frame: {
60 + kind: "response",
61 + method: "session/new",
62 + result: {
63 + sessionId,
64 + configOptions: [
65 + {
66 + id: "model",
67 + name: "Model",
68 + category: "model",
69 + type: "select",
70 + currentValue: "replay-model-a",
71 + options: [
72 + { value: "replay-model-a", name: "Model A" },
73 + { value: rejectedModel, name: "Model B" },
74 + ],
75 + },
76 + ],
77 + },
78 + },
79 + },
80 + ...(testCase.firstTurn
81 + ? recorded.entries
82 + .slice(sessionIndex + 1)
83 + .filter((entry) => entry.type !== "runtime_exit")
84 + : []),
85 + {
86 + type: "expect_outbound",
87 + label: "session.set_model",
88 + frame: {
89 + kind: "request",
90 + method: "session/set_config_option",
91 + params: { sessionId, configId: "model", value: rejectedModel },
92 + },
93 + },
94 + {
95 + type: "emit_inbound",
96 + label: "session.set_model.rejected",
97 + frame: {
98 + kind: "response",
99 + method: "session/set_config_option",
100 + error: {
101 + code: testCase.code,
102 + message: testCase.code === -32602 ? "Invalid provider" : "Authentication required",
103 + },
104 + },
105 + },
106 + ],
107 + } satisfies ProviderReplayTranscript;
108 + const materialized = yield* materializeFixtureInput({
109 + scenario: name,
110 + fixtureInput: {
111 + steps: Array.from({ length: testCase.firstTurn ? 2 : 1 }, () => ({
112 + type: "message" as const,
113 + text: SIMPLE_PROMPT,
114 + })),
115 + },
116 + driver: ProviderDriverKind.make("acpRegistry"),
117 + modelSelection: {
118 + ...modelSelection,
119 + model: testCase.firstTurn ? "replay-model-a" : rejectedModel,
120 + },
121 + });
122 + const scenario = {
123 + ...materialized,
124 + name,
125 + transcript: yield* AcpRegistryOrchestratorReplayHarness.decodeTranscript(
126 + materializeReplayTranscriptRuntimeInstructions(transcript, {
127 + driver: ProviderDriverKind.make("acpRegistry"),
128 + model: "replay-model-a",
129 + }),
130 + ),
131 + runtimePolicyOverride: { cwd: yield* checkpointWorkspace(name) },
132 + };
133 + yield* Effect.gen(function* () {
134 + const orchestrator = yield* Orchestrator.OrchestratorV2;
135 + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
136 + if (testCase.firstTurn) yield* EffectWorker.runDaemon.pipe(Effect.forkScoped);
137 + for (const [index, command] of materialized.commands.entries()) {
138 + const isFirstTurn =
139 + testCase.firstTurn && command.type === "message.dispatch" && index === 1;
140 + yield* orchestrator.dispatch(
141 + command.type === "message.dispatch" && !isFirstTurn
142 + ? { ...command, modelSelection }
143 + : command,
144 + );
145 + if (isFirstTurn) {
146 + // Wait for the committed terminal event before switching models.
147 + yield* orchestrator.streamStoredEvents.pipe(
148 + Stream.filter(
149 + ({ event }) => event.type === "run.updated" && event.payload.status === "completed",
150 + ),
151 + Stream.runHead,
152 + );
153 + }
154 + }
155 + if (testCase.firstTurn) {
156 + yield* orchestrator.streamStoredEvents.pipe(
157 + Stream.filter(
158 + ({ event }) => event.type === "run.updated" && event.payload.status === "failed",
159 + ),
160 + Stream.runHead,
161 + );
162 + } else {
163 + // Failed opens retry on the outbox schedule. Advance to each recorded
164 + // deadline, draining effects instead of polling the projection.
165 + for (let attempt = 0; attempt < 10; attempt++) {
166 + yield* worker.drain();
167 + const next = yield* worker.nextClaimableAt;
168 + if (Option.isNone(next)) break;
169 + const now = yield* DateTime.now;
170 + yield* TestClock.adjust(
171 + Math.max(0, DateTime.toEpochMillis(next.value) - DateTime.toEpochMillis(now)),
172 + );
173 + }
174 + }
175 + const projection = yield* orchestrator.getThreadProjection(
176 + materialized.projectionThreadIds[0]!,
177 + );
178 + assert.deepEqual(
179 + projection.runs.map((run) => run.status),
180 + testCase.firstTurn ? ["completed", "failed"] : ["failed"],
181 + );
182 + const errors = projection.turnItems.filter((item) => item.type === "error");
183 + assert.lengthOf(errors, 1);
184 + assert.equal(errors[0]?.failure.class, "provider_error");
185 + if (testCase.code === -32602) {
186 + assert.equal(
187 + errors[0]?.failure.message,
188 + 'The agent did not accept the model "replay-model-b". Choose another model for this provider, then retry.',
189 + );
190 + } else {
191 + assert.match(
192 + errors[0]?.failure.message ?? "",
193 + /^Failed to open acpRegistry provider session \S+\.$/u,
194 + );
195 + }
196 + }).pipe(
197 + Effect.provide(
198 + layerProviderReplay(scenario, AcpRegistryOrchestratorReplayHarness, {
199 + runEffectWorker: false,
200 + }),
201 + ),
202 + );
203 + }).pipe(
204 + Effect.scoped,
205 + provideDeterministicTestRuntime,
206 + Effect.provide(Layer.merge(IdAllocator.layer, NodeServices.layer)),
207 + ),
208 + );
209 +
210 + itRunsReplayFixtures("acpRegistry", (input) =>
211 + runReplayFixture({ ...input, harness: AcpRegistryOrchestratorReplayHarness }),
212 + );
213 +});
apps/server/src/orchestration-v2/testkit/ClaudeReplayFixtures.integration.test.ts
+226 -1
@@ -5,6 +5,7 @@ import {
5 isProviderNativeSubagentThread,
6 MessageId,
7 ProviderDriverKind,
8 + type ProviderReplayEntry,
9 } from "@t3tools/contracts";
10 import * as Effect from "effect/Effect";
11
@@ -17,7 +18,17 @@ import { subagentInput } from "./fixtures/subagent/input.ts";
18 import { runOrchestratorV2Scenario } from "./OrchestratorScenario.ts";
19 import * as ProviderReplayHarness from "./ProviderReplayHarness.ts";
20 import { materializeReplayTranscriptRuntimeInstructions } from "./ReplayRuntimeInstructions.ts";
20 -import { CLAUDE_MODEL_SELECTION, materializeFixtureInput } from "./fixtures/shared.ts";
21 +import {
22 + CLAUDE_MODEL_SELECTION,
23 + materializeFixtureInput,
24 + projectionFor,
25 +} from "./fixtures/shared.ts";
26 +import {
27 + findReplayFixture,
28 + itRunsReplayFixtures,
29 + runReplayFixture,
30 + type ReplayFixtureCase,
31 +} from "./ProviderReplayFixtures.ts";
32 import { readProviderReplayTranscript } from "@t3tools/provider-testing/replayTranscript";
33
34 const readTranscript = Effect.fn("readClaudeReplayFixture")(function* (file: URL) {
@@ -28,7 +39,13 @@ function readClaudeTranscriptFixture(path: string) {
39 return readTranscript(new URL(`./fixtures/${path}/claude_transcript.ndjson`, import.meta.url));
40 }
41
42 +function runClaudeReplayFixture(input: ReplayFixtureCase) {
43 + return runReplayFixture({ ...input, harness: ClaudeOrchestratorReplayHarness });
44 +}
45 +
46 describe("Claude Agent SDK replay fixtures", () => {
47 + itRunsReplayFixtures("claudeAgent", runClaudeReplayFixture);
48 +
49 it.effect("refuses messages to a native subagent thread without touching it", () =>
50 Effect.gen(function* () {
51 const raw = yield* readClaudeTranscriptFixture("subagent");
@@ -91,4 +108,212 @@ describe("Claude Agent SDK replay fixtures", () => {
108 );
109 }),
110 );
111 +
112 + const afterRootFixture = findReplayFixture("claude_background_subagent_after_root");
113 + const afterRootProvider = afterRootFixture?.providers[0];
114 + if (afterRootFixture !== undefined && afterRootProvider !== undefined) {
115 + const OPEN_TOOL_ID = "toolu_01QDa5jV5g1H9h6QyDfeJohD";
116 + const SUBAGENT_TASK_ID = "a2995bfced8019363";
117 +
118 + const moveOpenToolBeforeRootResult = (
119 + move: boolean,
120 + entries: ReadonlyArray<ProviderReplayEntry>,
121 + ): ReadonlyArray<ProviderReplayEntry> => {
122 + if (!move) return entries;
123 + const toolIndex = entries.findIndex(
124 + (entry) =>
125 + entry.type === "emit_inbound" &&
126 + (entry.frame as Record<string, any>).message?.content?.[0]?.id === OPEN_TOOL_ID,
127 + );
128 + const resultIndex = entries.findIndex(
129 + (entry) => entry.type === "emit_inbound" && entry.label === "result",
130 + );
131 + if (toolIndex < 0 || resultIndex < 0) throw new Error("transcript shape changed");
132 + const toolEntry = entries[toolIndex]!;
133 + const rest = entries.filter((_, index) => index !== toolIndex);
134 + return [...rest.slice(0, resultIndex), toolEntry, ...rest.slice(resultIndex)];
135 + };
136 + // A subagent's foreground Bash run during the root's turn is tailed like a
137 + // root call's, so the child's command row streams live.
138 + const SUBAGENT_BASH_ID = "toolu_01JMN3SSQCHzUTNua9AWdkiH";
139 + it.effect("streams a subagent's foreground Bash output live during the root turn", () =>
140 + runClaudeReplayFixture({
141 + fixtureName: afterRootFixture.name,
142 + buildInput: afterRootFixture.buildInput,
143 + driver: {
144 + ...afterRootProvider,
145 + assertOutput: (result) => {
146 + const childThreadId = projectionFor(result, afterRootFixture.name).subagents[0]
147 + ?.childThreadId;
148 + assert.exists(childThreadId);
149 + const command = result.projections
150 + .get(childThreadId)
151 + ?.turnItems.find(
152 + (item) =>
153 + item.type === "command_execution" &&
154 + JSON.stringify(item.input).includes("SUB_DONE_1"),
155 + );
156 + assert.exists(command);
157 + const streamed = result.commandOutput?.get(command.id);
158 + assert.exists(streamed, "the subagent's command must stream its output");
159 + assert.include(streamed.output.text, "SUB_DONE_1");
160 + },
161 + },
162 + transformTranscript: (transcript) => {
163 + const isBashFrame = (entry: ProviderReplayEntry) => {
164 + if (entry.type !== "emit_inbound") return false;
165 + const frame = entry.frame as Record<string, any>;
166 + return (
167 + frame.message?.content?.[0]?.id === SUBAGENT_BASH_ID ||
168 + frame.message?.content?.[0]?.tool_use_id === SUBAGENT_BASH_ID ||
169 + (frame.type === "system" && frame.tool_use_id === SUBAGENT_BASH_ID)
170 + );
171 + };
172 + // Recorded without the structured result the replay writes output files from.
173 + const bashFrames = transcript.entries.filter(isBashFrame).map((entry) => {
174 + const frame = (entry as { readonly frame: Record<string, any> }).frame;
175 + return frame.type === "user"
176 + ? {
177 + ...entry,
178 + frame: { ...frame, tool_use_result: { stdout: "SUB_DONE_1\n", stderr: "" } },
179 + }
180 + : entry;
181 + });
182 + const rest = transcript.entries.filter((entry) => !isBashFrame(entry));
183 + const resultIndex = rest.findIndex(
184 + (entry) => entry.type === "emit_inbound" && entry.label === "result",
185 + );
186 + assert.lengthOf(bashFrames, 4, "transcript shape changed");
187 + assert.isAtLeast(resultIndex, 0, "transcript shape changed");
188 + return {
189 + ...transcript,
190 + entries: [...rest.slice(0, resultIndex), ...bashFrames, ...rest.slice(resultIndex)],
191 + };
192 + },
193 + }),
194 + );
195 +
196 + // regression: #16486 a background subagent's tool left open while its parent is idle never settled
197 + it.effect.each([
198 + ["WebFetch", "while root is idle", "stopped"],
199 + ["Write", "before root settles", "failed"],
200 + ] as const)(
201 + "ends a background subagent's open %s (opened %s) when its notification is %s",
202 + ([tool, opens, notificationStatus]) =>
203 + runClaudeReplayFixture({
204 + fixtureName: afterRootFixture.name,
205 + buildInput: afterRootFixture.buildInput,
206 + driver: {
207 + ...afterRootProvider,
208 + assertOutput: (result) => {
209 + const parent = projectionFor(result, afterRootFixture.name);
210 + assert.deepEqual(
211 + parent.runs.map((run) => run.status),
212 + ["completed", "completed"],
213 + );
214 + const subagent = parent.subagents[0];
215 + assert.equal(
216 + subagent?.status,
217 + notificationStatus === "stopped" ? "cancelled" : "failed",
218 + );
219 + const childThreadId = subagent?.childThreadId;
220 + assert.exists(childThreadId);
221 + const child = result.projections.get(childThreadId);
222 + assert.exists(child);
223 + const fetches = child.turnItems.filter(
224 + (item) => item.type === (tool === "WebFetch" ? "web_search" : "file_change"),
225 + );
226 + assert.lengthOf(fetches, 1, `the ${tool} reached the child thread`);
227 + assert.deepEqual(
228 + {
229 + openTool: fetches.map((item) => item.status),
230 + openChildNodes: child.nodes
231 + .filter(
232 + (node) =>
233 + node.status === "running" ||
234 + node.status === "pending" ||
235 + node.status === "waiting",
236 + )
237 + .map((node) => `${node.kind}:${node.status}`),
238 + },
239 + {
240 + openTool: [notificationStatus === "stopped" ? "interrupted" : "failed"],
241 + openChildNodes: [],
242 + },
243 + );
244 + },
245 + },
246 + transformTranscript: (transcript) => ({
247 + ...transcript,
248 + entries: moveOpenToolBeforeRootResult(
249 + opens === "before root settles",
250 + transcript.entries.flatMap((entry): ReadonlyArray<ProviderReplayEntry> => {
251 + if (entry.type !== "emit_inbound") return [entry];
252 + const frame = entry.frame as Record<string, any>;
253 + if (
254 + frame.type === "assistant" &&
255 + frame.message?.content?.[0]?.id === OPEN_TOOL_ID
256 + ) {
257 + return [
258 + {
259 + ...entry,
260 + frame: {
261 + ...frame,
262 + message: {
263 + ...frame.message,
264 + content: [
265 + {
266 + ...frame.message.content[0],
267 + name: tool,
268 + input:
269 + tool === "WebFetch"
270 + ? { url: "https://example.com", prompt: "Summarize" }
271 + : { file_path: "notes.txt", content: "SUB_DONE_2" },
272 + },
273 + ],
274 + },
275 + },
276 + },
277 + ];
278 + }
279 + if (
280 + (frame.type === "system" &&
281 + (frame.subtype === "task_started" || frame.subtype === "task_notification") &&
282 + frame.tool_use_id === OPEN_TOOL_ID) ||
283 + (frame.type === "user" &&
284 + frame.message?.content?.[0]?.tool_use_id === OPEN_TOOL_ID) ||
285 + (frame.type === "assistant" &&
286 + frame.parent_tool_use_id !== null &&
287 + frame.message?.content?.[0]?.text === "SUB_FINAL_REPORT")
288 + ) {
289 + return [];
290 + }
291 + if (frame.type === "system" && frame.subtype === "task_updated") {
292 + return [
293 + {
294 + ...entry,
295 + frame: {
296 + ...frame,
297 + patch: {
298 + ...frame.patch,
299 + status: notificationStatus === "stopped" ? "killed" : "failed",
300 + },
301 + },
302 + },
303 + ];
304 + }
305 + if (
306 + frame.type === "system" &&
307 + frame.subtype === "task_notification" &&
308 + frame.task_id === SUBAGENT_TASK_ID
309 + ) {
310 + return [{ ...entry, frame: { ...frame, status: notificationStatus } }];
311 + }
312 + return [entry];
313 + }),
314 + ),
315 + }),
316 + }),
317 + );
318 + }
319 });
apps/server/src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts new
+17
@@ -0,0 +1,17 @@
1 +import { describe } from "@effect/vitest";
2 +import * as Effect from "effect/Effect";
3 +
4 +import { materializeReplayTranscriptWorkspace } from "@t3tools/provider-testing/replayTranscript";
5 +import { CodexOrchestratorReplayHarness } from "../Adapters/CodexAdapterV2.testkit.ts";
6 +import { itRunsReplayFixtures, runReplayFixture } from "./ProviderReplayFixtures.ts";
7 +
8 +describe("Codex replay fixtures", () => {
9 + itRunsReplayFixtures("codex", (input) =>
10 + runReplayFixture({
11 + ...input,
12 + harness: CodexOrchestratorReplayHarness,
13 + materializeWorkspace: (transcript, workspace) =>
14 + Effect.succeed(materializeReplayTranscriptWorkspace(transcript, workspace)),
15 + }),
16 + );
17 +});
apps/server/src/orchestration-v2/testkit/CursorReplayFixtures.integration.test.ts new
+29
@@ -0,0 +1,29 @@
1 +import { describe, it } from "@effect/vitest";
2 +
3 +import { CursorOrchestratorReplayHarness } from "../Adapters/CursorAdapterV2.testkit.ts";
4 +import { messageRestartInput } from "./fixtures/message_steering/input.ts";
5 +import {
6 + findReplayFixture,
7 + itRunsReplayFixtures,
8 + runReplayFixture,
9 +} from "./ProviderReplayFixtures.ts";
10 +
11 +describe("Cursor replay fixtures", () => {
12 + itRunsReplayFixtures("cursor", (input) =>
13 + runReplayFixture({ ...input, harness: CursorOrchestratorReplayHarness }),
14 + );
15 +
16 + const cursorSteeringProvider = findReplayFixture("message_steering")?.providers.find(
17 + (provider) => provider.driver === "cursor",
18 + );
19 + if (cursorSteeringProvider !== undefined) {
20 + it.effect("executes explicit Cursor restart_active through the recorded SDK boundary", () =>
21 + runReplayFixture({
22 + fixtureName: "message_steering",
23 + buildInput: messageRestartInput,
24 + driver: cursorSteeringProvider,
25 + harness: CursorOrchestratorReplayHarness,
26 + }),
27 + );
28 + }
29 +});
apps/server/src/orchestration-v2/testkit/GrokReplayFixtures.integration.test.ts new
+10
@@ -0,0 +1,10 @@
1 +import { describe } from "@effect/vitest";
2 +
3 +import { GrokOrchestratorReplayHarness } from "../Adapters/GrokAdapterV2.testkit.ts";
4 +import { itRunsReplayFixtures, runReplayFixture } from "./ProviderReplayFixtures.ts";
5 +
6 +describe("Grok replay fixtures", () => {
7 + itRunsReplayFixtures("grok", (input) =>
8 + runReplayFixture({ ...input, harness: GrokOrchestratorReplayHarness }),
9 + );
10 +});
apps/server/src/orchestration-v2/testkit/MuseReplayFixtures.integration.test.ts new
+17
@@ -0,0 +1,17 @@
1 +import { describe } from "@effect/vitest";
2 +
3 +import {
4 + materializeMuseReplayWorkspace,
5 + MuseOrchestratorReplayHarness,
6 +} from "../Adapters/MuseAdapterV2.testkit.ts";
7 +import { itRunsReplayFixtures, runReplayFixture } from "./ProviderReplayFixtures.ts";
8 +
9 +describe("Muse replay fixtures", () => {
10 + itRunsReplayFixtures("muse", (input) =>
11 + runReplayFixture({
12 + ...input,
13 + harness: MuseOrchestratorReplayHarness,
14 + materializeWorkspace: materializeMuseReplayWorkspace,
15 + }),
16 + );
17 +});
apps/server/src/orchestration-v2/testkit/OpenCodeReplayFixtures.integration.test.ts new
+68
@@ -0,0 +1,68 @@
1 +import { describe, it } from "@effect/vitest";
2 +import type { ProviderReplayEntry } from "@t3tools/contracts";
3 +import * as Effect from "effect/Effect";
4 +
5 +import { OpenCodeOrchestratorReplayHarness } from "../Adapters/OpenCodeAdapterV2.testkit.ts";
6 +import {
7 + OPENCODE2_HTTP_PROTOCOL,
8 + OpenCode2OrchestratorReplayHarness,
9 +} from "../Adapters/OpenCode2AdapterV2.testkit.ts";
10 +import {
11 + findReplayFixture,
12 + itRunsReplayFixtures,
13 + readReplayFixtureTranscript,
14 + runReplayFixture,
15 + type ReplayFixtureCase,
16 +} from "./ProviderReplayFixtures.ts";
17 +
18 +// OpenCode fixtures are recorded against either the v1 or the v2 HTTP protocol.
19 +function runOpenCodeReplayFixture(input: ReplayFixtureCase) {
20 + return readReplayFixtureTranscript(input.driver.transcriptFile).pipe(
21 + Effect.flatMap((transcript) =>
22 + transcript.protocol === OPENCODE2_HTTP_PROTOCOL
23 + ? runReplayFixture({ ...input, harness: OpenCode2OrchestratorReplayHarness })
24 + : runReplayFixture({ ...input, harness: OpenCodeOrchestratorReplayHarness }),
25 + ),
26 + );
27 +}
28 +
29 +describe("OpenCode replay fixtures", () => {
30 + itRunsReplayFixtures("opencode", runOpenCodeReplayFixture);
31 +
32 + it.effect.each(
33 + (
34 + [
35 + ["opencode2_subagent", "session.execution.started.2"],
36 + ["opencode2_background", "session.execution.started.3"],
37 + ] as const
38 + ).flatMap(([fixtureName, label]) => {
39 + const fixture = findReplayFixture(fixtureName);
40 + const provider = fixture?.providers[0];
41 + return fixture === undefined || provider === undefined
42 + ? []
43 + : [[fixtureName, label, fixture, provider] as const];
44 + }),
45 + )(
46 + "runs %s when %s is an execution start this build cannot decode",
47 + ([fixtureName, label, fixture, provider]) =>
48 + runOpenCodeReplayFixture({
49 + fixtureName,
50 + buildInput: fixture.buildInput,
51 + driver: provider,
52 + transformTranscript: (transcript) => ({
53 + ...transcript,
54 + entries: transcript.entries.map((entry) =>
55 + entry.type === "emit_inbound" && entry.label === label
56 + ? undecodableEvent(entry)
57 + : entry,
58 + ),
59 + }),
60 + }),
61 + );
62 +});
63 +
64 +function undecodableEvent(entry: ProviderReplayEntry): ProviderReplayEntry {
65 + if (entry.type !== "emit_inbound") return entry;
66 + const frame = entry.frame as { readonly event: Record<string, unknown> };
67 + return { ...entry, frame: { ...frame, event: { ...frame.event, durable: "not-an-envelope" } } };
68 +}
apps/server/src/orchestration-v2/testkit/OrchestratorReplayFixtures.integration.test.ts deleted
-704
@@ -1,704 +0,0 @@
1 -import * as NodeServices from "@effect/platform-node/NodeServices";
2 -import { assert, describe, it } from "@effect/vitest";
3 -import {
4 - ProviderDriverKind,
5 - ProviderInstanceId,
6 - type OrchestrationV2DomainEvent,
7 - type ProviderReplayEntry,
8 - type ProviderReplayTranscript,
9 -} from "@t3tools/contracts";
10 -import * as DateTime from "effect/DateTime";
11 -import * as Effect from "effect/Effect";
12 -import * as FileSystem from "effect/FileSystem";
13 -import * as Layer from "effect/Layer";
14 -import * as Option from "effect/Option";
15 -import * as Path from "effect/Path";
16 -import * as Stream from "effect/Stream";
17 -import { TestClock } from "effect/testing";
18 -
19 -import * as EffectWorker from "../EffectWorker.ts";
20 -import * as Orchestrator from "../Orchestrator.ts";
21 -
22 -import { ClaudeOrchestratorReplayHarness } from "../Adapters/ClaudeAdapterV2.testkit.ts";
23 -import { CodexOrchestratorReplayHarness } from "../Adapters/CodexAdapterV2.testkit.ts";
24 -import { CursorOrchestratorReplayHarness } from "../Adapters/CursorAdapterV2.testkit.ts";
25 -import { AcpRegistryOrchestratorReplayHarness } from "../Adapters/AcpRegistryAdapterV2.testkit.ts";
26 -import { GrokOrchestratorReplayHarness } from "../Adapters/GrokAdapterV2.testkit.ts";
27 -import { OpenCodeOrchestratorReplayHarness } from "../Adapters/OpenCodeAdapterV2.testkit.ts";
28 -import {
29 - OPENCODE2_HTTP_PROTOCOL,
30 - OpenCode2OrchestratorReplayHarness,
31 -} from "../Adapters/OpenCode2AdapterV2.testkit.ts";
32 -import {
33 - materializeMuseReplayWorkspace,
34 - MuseOrchestratorReplayHarness,
35 -} from "../Adapters/MuseAdapterV2.testkit.ts";
36 -import { PiOrchestratorReplayHarness } from "../Adapters/PiAdapterV2.testkit.ts";
37 -import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
38 -import { provideDeterministicTestRuntime } from "./DeterministicRuntime.ts";
39 -import { ORCHESTRATOR_REPLAY_FIXTURES } from "./fixtures/index.ts";
40 -import { messageRestartInput } from "./fixtures/message_steering/input.ts";
41 -import {
42 - assertProviderNativeSubagentRootTurns,
43 - materializeFixtureInput,
44 - SIMPLE_PROMPT,
45 - projectionFor,
46 - type OrchestratorFixtureInput,
47 - type ProviderOrchestratorReplayVariant,
48 -} from "./fixtures/shared.ts";
49 -import {
50 - runOrchestratorV2ProviderReplayScenario,
51 - layerProviderReplay,
52 - type OrchestratorV2ProviderReplayHarness,
53 -} from "./ProviderReplayHarness.ts";
54 -import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace";
55 -import { materializeReplayTranscriptRuntimeInstructions } from "./ReplayRuntimeInstructions.ts";
56 -import {
57 - materializeReplayTranscriptWorkspace,
58 - readProviderReplayTranscript,
59 -} from "@t3tools/provider-testing/replayTranscript";
60 -
61 -const readTranscript = Effect.fn("readOrchestratorReplayTranscript")(function* (file: URL) {
62 - return yield* readProviderReplayTranscript(file);
63 -}, Effect.provide(NodeServices.layer));
64 -
65 -function normalizeTestError(cause: unknown): Error {
66 - return cause instanceof Error ? cause : new Error(String(cause));
67 -}
68 -
69 -function transcriptEntriesThroughLabel(
70 - transcript: ProviderReplayTranscript,
71 - label: string | undefined,
72 -): ProviderReplayTranscript {
73 - if (label === undefined) {
74 - return transcript;
75 - }
76 - const entryIndex = transcript.entries.findIndex(
77 - (entry) => entry.type === "emit_inbound" && entry.label === label,
78 - );
79 - if (entryIndex === -1) {
80 - throw new Error(`${transcript.scenario} is missing inbound transcript label ${label}.`);
81 - }
82 - return {
83 - ...transcript,
84 - entries: transcript.entries.slice(0, entryIndex + 1),
85 - };
86 -}
87 -
88 -function isStreamingAssistantEvent(event: OrchestrationV2DomainEvent): boolean {
89 - switch (event.type) {
90 - case "node.updated":
91 - return event.payload.kind === "assistant_message" && event.payload.status === "running";
92 - case "message.updated":
93 - return event.payload.role === "assistant" && event.payload.streaming;
94 - case "turn-item.updated":
95 - return event.payload.type === "assistant_message" && event.payload.streaming;
96 - default:
97 - return false;
98 - }
99 -}
100 -
101 -const runFixtureProvider = Effect.fn("runOrchestratorReplayFixture")(function* <
102 - Transcript extends ProviderReplayTranscript,
103 - Error,
104 ->(input: {
105 - readonly fixtureName: string;
106 - readonly buildInput: () => OrchestratorFixtureInput;
107 - readonly driver: ProviderOrchestratorReplayVariant;
108 - readonly harness: OrchestratorV2ProviderReplayHarness<Transcript, Error>;
109 - readonly transformTranscript?: (transcript: ProviderReplayTranscript) => ProviderReplayTranscript;
110 -}) {
111 - const recordedTranscript = yield* readTranscript(input.driver.transcriptFile);
112 - const rawTranscript = input.transformTranscript?.(recordedTranscript) ?? recordedTranscript;
113 - const replayTranscript = materializeReplayTranscriptRuntimeInstructions(
114 - transcriptEntriesThroughLabel(rawTranscript, input.driver.transcriptEntriesThroughLabel),
115 - { driver: input.driver.driver, model: input.driver.modelSelection.model },
116 - );
117 - const fixtureInput = input.buildInput();
118 - const workspace = yield* checkpointWorkspace(input.fixtureName, fixtureInput.workspaceFiles);
119 - const transcript = yield* input.harness.decodeTranscript(
120 - input.driver.driver === "codex"
121 - ? materializeReplayTranscriptWorkspace(replayTranscript, workspace)
122 - : input.driver.driver === "muse"
123 - ? yield* materializeMuseReplayWorkspace(replayTranscript, workspace)
124 - : replayTranscript,
125 - );
126 - const materialized = yield* materializeFixtureInput({
127 - scenario: input.fixtureName,
128 - fixtureInput,
129 - driver: input.driver.driver,
130 - modelSelection: input.driver.modelSelection,
131 - }).pipe(Effect.provide(IdAllocator.layer), provideDeterministicTestRuntime);
132 - const scenario = {
133 - name: `${input.fixtureName}/${input.driver.driver}`,
134 - transcript,
135 - commands: materialized.commands,
136 - steps: materialized.steps,
137 - projectionThreadIds: materialized.projectionThreadIds,
138 - runtimePolicyOverride: {
139 - ...input.driver.runtimePolicyOverride,
140 - cwd: workspace,
141 - },
142 - };
143 -
144 - const result = yield* runOrchestratorV2ProviderReplayScenario(
145 - scenario,
146 - input.harness,
147 - input.driver.runContinuationWorker === true ? { runContinuationWorker: true } : {},
148 - ).pipe(provideDeterministicTestRuntime);
149 - input.driver.assertOutput(result, transcript);
150 - assertProviderNativeSubagentRootTurns(result);
151 - const expectedAbsentWorkspacePaths = input.driver.expectedAbsentWorkspacePaths;
152 - if (expectedAbsentWorkspacePaths !== undefined) {
153 - yield* Effect.gen(function* () {
154 - const fs = yield* FileSystem.FileSystem;
155 - const path = yield* Path.Path;
156 - for (const relativePath of expectedAbsentWorkspacePaths) {
157 - assert.isFalse(
158 - yield* fs.exists(path.join(workspace, relativePath)),
159 - `${input.fixtureName}/${input.driver.driver} must not create ${relativePath} in the replay workspace`,
160 - );
161 - }
162 - }).pipe(Effect.provide(NodeServices.layer));
163 - }
164 - assert.isFalse(
165 - result.domainEvents.some(isStreamingAssistantEvent),
166 - "buffered delivery must not persist streaming assistant artifacts",
167 - );
168 - const projectionThreadId = materialized.projectionThreadIds[0];
169 - assert.isDefined(projectionThreadId);
170 - const projection = result.projections.get(projectionThreadId);
171 - assert.isDefined(projection);
172 - const latestRun = projection.runs.at(-1);
173 - assert.deepEqual(latestRun?.modelSelection, input.driver.modelSelection);
174 - if (projection.runs.some((run) => run.status === "completed")) {
175 - const threadStartCheckpoint = projection.checkpoints.find(
176 - (checkpoint) => checkpoint.ordinalWithinScope === 0 && checkpoint.appRunOrdinal === null,
177 - );
178 - assert.isDefined(
179 - threadStartCheckpoint,
180 - "completed threads must retain an addressable thread-start checkpoint",
181 - );
182 - assert.equal(threadStartCheckpoint.status, "ready");
183 - }
184 - return result;
185 -});
186 -
187 -function runFixtureProviderWithRegisteredHarness(input: {
188 - readonly fixtureName: string;
189 - readonly buildInput: () => OrchestratorFixtureInput;
190 - readonly driver: ProviderOrchestratorReplayVariant;
191 - readonly transformTranscript?: (transcript: ProviderReplayTranscript) => ProviderReplayTranscript;
192 -}) {
193 - switch (input.driver.driver) {
194 - case "codex":
195 - return runFixtureProvider({
196 - ...input,
197 - harness: CodexOrchestratorReplayHarness,
198 - }).pipe(Effect.mapError(normalizeTestError), Effect.scoped);
199 - case "claudeAgent":
200 - return runFixtureProvider({
201 - ...input,
202 - harness: ClaudeOrchestratorReplayHarness,
203 - }).pipe(Effect.mapError(normalizeTestError), Effect.scoped);
204 - case "cursor":
205 - return runFixtureProvider({
206 - ...input,
207 - harness: CursorOrchestratorReplayHarness,
208 - }).pipe(Effect.mapError(normalizeTestError), Effect.scoped);
209 - case "grok":
210 - return runFixtureProvider({
211 - ...input,
212 - harness: GrokOrchestratorReplayHarness,
213 - }).pipe(Effect.mapError(normalizeTestError), Effect.scoped);
214 - case "acpRegistry":
215 - return runFixtureProvider({
216 - ...input,
217 - harness: AcpRegistryOrchestratorReplayHarness,
218 - }).pipe(Effect.mapError(normalizeTestError), Effect.scoped);
219 - case "opencode":
220 - return readTranscript(input.driver.transcriptFile).pipe(
221 - Effect.flatMap((transcript) =>
222 - transcript.protocol === OPENCODE2_HTTP_PROTOCOL
223 - ? runFixtureProvider({ ...input, harness: OpenCode2OrchestratorReplayHarness })
224 - : runFixtureProvider({ ...input, harness: OpenCodeOrchestratorReplayHarness }),
225 - ),
226 - Effect.mapError(normalizeTestError),
227 - Effect.scoped,
228 - );
229 - case "pi":
230 - return runFixtureProvider({
231 - ...input,
232 - harness: PiOrchestratorReplayHarness,
233 - }).pipe(Effect.mapError(normalizeTestError), Effect.scoped);
234 - case "muse":
235 - return runFixtureProvider({
236 - ...input,
237 - harness: MuseOrchestratorReplayHarness,
238 - }).pipe(Effect.mapError(normalizeTestError), Effect.scoped);
239 - default:
240 - return Effect.die(
241 - new Error(`No replay harness registered for provider ${input.driver.driver}.`),
242 - );
243 - }
244 -}
245 -
246 -describe("orchestrator replay fixtures", () => {
247 - it.effect.each([
248 - { name: "open", firstTurn: false, code: -32602 },
249 - { name: "switch", firstTurn: true, code: -32602 },
250 - { name: "authentication", firstTurn: false, code: -32000 },
251 - ])("preserves the ACP model-write $name failure through orchestration", (testCase) =>
252 - Effect.gen(function* () {
253 - const name = `acp_model_write_${testCase.name}`;
254 - const sessionId = "grok-replay-session-1";
255 - const rejectedModel = "replay-model-b";
256 - const modelSelection = {
257 - instanceId: ProviderInstanceId.make("acpRegistry"),
258 - model: rejectedModel,
259 - };
260 - const recorded = yield* readTranscript(
261 - new URL("./fixtures/simple/registry_transcript.ndjson", import.meta.url),
262 - );
263 - const sessionIndex = recorded.entries.findIndex(
264 - (entry) => "label" in entry && entry.label === "session.created",
265 - );
266 - assert.isAtLeast(sessionIndex, 0);
267 - const transcript = {
268 - ...recorded,
269 - scenario: name,
270 - entries: [
271 - ...recorded.entries.slice(0, sessionIndex),
272 - {
273 - type: "emit_inbound",
274 - label: "session.created",
275 - frame: {
276 - kind: "response",
277 - method: "session/new",
278 - result: {
279 - sessionId,
280 - configOptions: [
281 - {
282 - id: "model",
283 - name: "Model",
284 - category: "model",
285 - type: "select",
286 - currentValue: "replay-model-a",
287 - options: [
288 - { value: "replay-model-a", name: "Model A" },
289 - { value: rejectedModel, name: "Model B" },
290 - ],
291 - },
292 - ],
293 - },
294 - },
295 - },
296 - ...(testCase.firstTurn
297 - ? recorded.entries
298 - .slice(sessionIndex + 1)
299 - .filter((entry) => entry.type !== "runtime_exit")
300 - : []),
301 - {
302 - type: "expect_outbound",
303 - label: "session.set_model",
304 - frame: {
305 - kind: "request",
306 - method: "session/set_config_option",
307 - params: { sessionId, configId: "model", value: rejectedModel },
308 - },
309 - },
310 - {
311 - type: "emit_inbound",
312 - label: "session.set_model.rejected",
313 - frame: {
314 - kind: "response",
315 - method: "session/set_config_option",
316 - error: {
317 - code: testCase.code,
318 - message: testCase.code === -32602 ? "Invalid provider" : "Authentication required",
319 - },
320 - },
321 - },
322 - ],
323 - } satisfies ProviderReplayTranscript;
324 - const materialized = yield* materializeFixtureInput({
325 - scenario: name,
326 - fixtureInput: {
327 - steps: Array.from({ length: testCase.firstTurn ? 2 : 1 }, () => ({
328 - type: "message" as const,
329 - text: SIMPLE_PROMPT,
330 - })),
331 - },
332 - driver: ProviderDriverKind.make("acpRegistry"),
333 - modelSelection: {
334 - ...modelSelection,
335 - model: testCase.firstTurn ? "replay-model-a" : rejectedModel,
336 - },
337 - });
338 - const scenario = {
339 - ...materialized,
340 - name,
341 - transcript: yield* AcpRegistryOrchestratorReplayHarness.decodeTranscript(
342 - materializeReplayTranscriptRuntimeInstructions(transcript, {
343 - driver: ProviderDriverKind.make("acpRegistry"),
344 - model: "replay-model-a",
345 - }),
346 - ),
347 - runtimePolicyOverride: { cwd: yield* checkpointWorkspace(name) },
348 - };
349 - yield* Effect.gen(function* () {
350 - const orchestrator = yield* Orchestrator.OrchestratorV2;
351 - const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
352 - if (testCase.firstTurn) yield* EffectWorker.runDaemon.pipe(Effect.forkScoped);
353 - for (const [index, command] of materialized.commands.entries()) {
354 - const isFirstTurn =
355 - testCase.firstTurn && command.type === "message.dispatch" && index === 1;
356 - yield* orchestrator.dispatch(
357 - command.type === "message.dispatch" && !isFirstTurn
358 - ? { ...command, modelSelection }
359 - : command,
360 - );
361 - if (isFirstTurn) {
362 - // Wait for the committed terminal event before switching models.
363 - yield* orchestrator.streamStoredEvents.pipe(
364 - Stream.filter(
365 - ({ event }) => event.type === "run.updated" && event.payload.status === "completed",
366 - ),
367 - Stream.runHead,
368 - );
369 - }
370 - }
371 - if (testCase.firstTurn) {
372 - yield* orchestrator.streamStoredEvents.pipe(
373 - Stream.filter(
374 - ({ event }) => event.type === "run.updated" && event.payload.status === "failed",
375 - ),
376 - Stream.runHead,
377 - );
378 - } else {
379 - // Failed opens retry on the outbox schedule. Advance to each recorded
380 - // deadline, draining effects instead of polling the projection.
381 - for (let attempt = 0; attempt < 10; attempt++) {
382 - yield* worker.drain();
383 - const next = yield* worker.nextClaimableAt;
384 - if (Option.isNone(next)) break;
385 - const now = yield* DateTime.now;
386 - yield* TestClock.adjust(
387 - Math.max(0, DateTime.toEpochMillis(next.value) - DateTime.toEpochMillis(now)),
388 - );
389 - }
390 - }
391 - const projection = yield* orchestrator.getThreadProjection(
392 - materialized.projectionThreadIds[0]!,
393 - );
394 - assert.deepEqual(
395 - projection.runs.map((run) => run.status),
396 - testCase.firstTurn ? ["completed", "failed"] : ["failed"],
397 - );
398 - const errors = projection.turnItems.filter((item) => item.type === "error");
399 - assert.lengthOf(errors, 1);
400 - assert.equal(errors[0]?.failure.class, "provider_error");
401 - if (testCase.code === -32602) {
402 - assert.equal(
403 - errors[0]?.failure.message,
404 - 'The agent did not accept the model "replay-model-b". Choose another model for this provider, then retry.',
405 - );
406 - } else {
407 - assert.match(
408 - errors[0]?.failure.message ?? "",
409 - /^Failed to open acpRegistry provider session \S+\.$/u,
410 - );
411 - }
412 - }).pipe(
413 - Effect.provide(
414 - layerProviderReplay(scenario, AcpRegistryOrchestratorReplayHarness, {
415 - runEffectWorker: false,
416 - }),
417 - ),
418 - );
419 - }).pipe(
420 - Effect.scoped,
421 - provideDeterministicTestRuntime,
422 - Effect.provide(Layer.merge(IdAllocator.layer, NodeServices.layer)),
423 - ),
424 - );
425 -
426 - it.effect.each(
427 - ORCHESTRATOR_REPLAY_FIXTURES.flatMap((fixture) =>
428 - fixture.providers.map(
429 - (provider) => [fixture.name, provider.driver, fixture, provider] as const,
430 - ),
431 - ),
432 - )("runs %s/%s through OrchestratorV2 using deterministic replay", ([, , fixture, provider]) =>
433 - runFixtureProviderWithRegisteredHarness({
434 - fixtureName: fixture.name,
435 - buildInput: fixture.buildInput,
436 - driver: provider,
437 - }),
438 - );
439 -
440 - const steeringFixture = ORCHESTRATOR_REPLAY_FIXTURES.find(
441 - (fixture) => fixture.name === "message_steering",
442 - );
443 - const cursorSteeringProvider = steeringFixture?.providers.find(
444 - (provider) => provider.driver === "cursor",
445 - );
446 - if (cursorSteeringProvider !== undefined) {
447 - it.effect("executes explicit Cursor restart_active through the recorded SDK boundary", () =>
448 - runFixtureProviderWithRegisteredHarness({
449 - fixtureName: "message_steering",
450 - buildInput: messageRestartInput,
451 - driver: cursorSteeringProvider,
452 - }),
453 - );
454 - }
455 -
456 - it.effect.each(
457 - (
458 - [
459 - ["opencode2_subagent", "session.execution.started.2"],
460 - ["opencode2_background", "session.execution.started.3"],
461 - ] as const
462 - ).flatMap(([fixtureName, label]) => {
463 - const fixture = ORCHESTRATOR_REPLAY_FIXTURES.find(
464 - (candidate) => candidate.name === fixtureName,
465 - );
466 - const provider = fixture?.providers[0];
467 - return fixture === undefined || provider === undefined
468 - ? []
469 - : [[fixtureName, label, fixture, provider] as const];
470 - }),
471 - )(
472 - "runs %s when %s is an execution start this build cannot decode",
473 - ([fixtureName, label, fixture, provider]) =>
474 - runFixtureProviderWithRegisteredHarness({
475 - fixtureName,
476 - buildInput: fixture.buildInput,
477 - driver: provider,
478 - transformTranscript: (transcript) => ({
479 - ...transcript,
480 - entries: transcript.entries.map((entry) =>
481 - entry.type === "emit_inbound" && entry.label === label
482 - ? undecodableEvent(entry)
483 - : entry,
484 - ),
485 - }),
486 - }),
487 - );
488 -
489 - const afterRootFixture = ORCHESTRATOR_REPLAY_FIXTURES.find(
490 - (candidate) => candidate.name === "claude_background_subagent_after_root",
491 - );
492 - const afterRootProvider = afterRootFixture?.providers[0];
493 - if (afterRootFixture !== undefined && afterRootProvider !== undefined) {
494 - const OPEN_TOOL_ID = "toolu_01QDa5jV5g1H9h6QyDfeJohD";
495 - const SUBAGENT_TASK_ID = "a2995bfced8019363";
496 -
497 - const moveOpenToolBeforeRootResult = (
498 - move: boolean,
499 - entries: ReadonlyArray<ProviderReplayEntry>,
500 - ): ReadonlyArray<ProviderReplayEntry> => {
501 - if (!move) return entries;
502 - const toolIndex = entries.findIndex(
503 - (entry) =>
504 - entry.type === "emit_inbound" &&
505 - (entry.frame as Record<string, any>).message?.content?.[0]?.id === OPEN_TOOL_ID,
506 - );
507 - const resultIndex = entries.findIndex(
508 - (entry) => entry.type === "emit_inbound" && entry.label === "result",
509 - );
510 - if (toolIndex < 0 || resultIndex < 0) throw new Error("transcript shape changed");
511 - const toolEntry = entries[toolIndex]!;
512 - const rest = entries.filter((_, index) => index !== toolIndex);
513 - return [...rest.slice(0, resultIndex), toolEntry, ...rest.slice(resultIndex)];
514 - };
515 - // A subagent's foreground Bash run during the root's turn is tailed like a
516 - // root call's, so the child's command row streams live.
517 - const SUBAGENT_BASH_ID = "toolu_01JMN3SSQCHzUTNua9AWdkiH";
518 - it.effect("streams a subagent's foreground Bash output live during the root turn", () =>
519 - runFixtureProviderWithRegisteredHarness({
520 - fixtureName: afterRootFixture.name,
521 - buildInput: afterRootFixture.buildInput,
522 - driver: {
523 - ...afterRootProvider,
524 - assertOutput: (result) => {
525 - const childThreadId = projectionFor(result, afterRootFixture.name).subagents[0]
526 - ?.childThreadId;
527 - assert.exists(childThreadId);
528 - const command = result.projections
529 - .get(childThreadId)
530 - ?.turnItems.find(
531 - (item) =>
532 - item.type === "command_execution" &&
533 - JSON.stringify(item.input).includes("SUB_DONE_1"),
534 - );
535 - assert.exists(command);
536 - const streamed = result.commandOutput?.get(command.id);
537 - assert.exists(streamed, "the subagent's command must stream its output");
538 - assert.include(streamed.output.text, "SUB_DONE_1");
539 - },
540 - },
541 - transformTranscript: (transcript) => {
542 - const isBashFrame = (entry: ProviderReplayEntry) => {
543 - if (entry.type !== "emit_inbound") return false;
544 - const frame = entry.frame as Record<string, any>;
545 - return (
546 - frame.message?.content?.[0]?.id === SUBAGENT_BASH_ID ||
547 - frame.message?.content?.[0]?.tool_use_id === SUBAGENT_BASH_ID ||
548 - (frame.type === "system" && frame.tool_use_id === SUBAGENT_BASH_ID)
549 - );
550 - };
551 - // Recorded without the structured result the replay writes output files from.
552 - const bashFrames = transcript.entries.filter(isBashFrame).map((entry) => {
553 - const frame = (entry as { readonly frame: Record<string, any> }).frame;
554 - return frame.type === "user"
555 - ? {
556 - ...entry,
557 - frame: { ...frame, tool_use_result: { stdout: "SUB_DONE_1\n", stderr: "" } },
558 - }
559 - : entry;
560 - });
561 - const rest = transcript.entries.filter((entry) => !isBashFrame(entry));
562 - const resultIndex = rest.findIndex(
563 - (entry) => entry.type === "emit_inbound" && entry.label === "result",
564 - );
565 - assert.lengthOf(bashFrames, 4, "transcript shape changed");
566 - assert.isAtLeast(resultIndex, 0, "transcript shape changed");
567 - return {
568 - ...transcript,
569 - entries: [...rest.slice(0, resultIndex), ...bashFrames, ...rest.slice(resultIndex)],
570 - };
571 - },
572 - }),
573 - );
574 -
575 - // regression: #16486 a background subagent's tool left open while its parent is idle never settled
576 - it.effect.each([
577 - ["WebFetch", "while root is idle", "stopped"],
578 - ["Write", "before root settles", "failed"],
579 - ] as const)(
580 - "ends a background subagent's open %s (opened %s) when its notification is %s",
581 - ([tool, opens, notificationStatus]) =>
582 - runFixtureProviderWithRegisteredHarness({
583 - fixtureName: afterRootFixture.name,
584 - buildInput: afterRootFixture.buildInput,
585 - driver: {
586 - ...afterRootProvider,
587 - assertOutput: (result) => {
588 - const parent = projectionFor(result, afterRootFixture.name);
589 - assert.deepEqual(
590 - parent.runs.map((run) => run.status),
591 - ["completed", "completed"],
592 - );
593 - const subagent = parent.subagents[0];
594 - assert.equal(
595 - subagent?.status,
596 - notificationStatus === "stopped" ? "cancelled" : "failed",
597 - );
598 - const childThreadId = subagent?.childThreadId;
599 - assert.exists(childThreadId);
600 - const child = result.projections.get(childThreadId);
601 - assert.exists(child);
602 - const fetches = child.turnItems.filter(
603 - (item) => item.type === (tool === "WebFetch" ? "web_search" : "file_change"),
604 - );
605 - assert.lengthOf(fetches, 1, `the ${tool} reached the child thread`);
606 - assert.deepEqual(
607 - {
608 - openTool: fetches.map((item) => item.status),
609 - openChildNodes: child.nodes
610 - .filter(
611 - (node) =>
612 - node.status === "running" ||
613 - node.status === "pending" ||
614 - node.status === "waiting",
615 - )
616 - .map((node) => `${node.kind}:${node.status}`),
617 - },
618 - {
619 - openTool: [notificationStatus === "stopped" ? "interrupted" : "failed"],
620 - openChildNodes: [],
621 - },
622 - );
623 - },
624 - },
625 - transformTranscript: (transcript) => ({
626 - ...transcript,
627 - entries: moveOpenToolBeforeRootResult(
628 - opens === "before root settles",
629 - transcript.entries.flatMap((entry): ReadonlyArray<ProviderReplayEntry> => {
630 - if (entry.type !== "emit_inbound") return [entry];
631 - const frame = entry.frame as Record<string, any>;
632 - if (
633 - frame.type === "assistant" &&
634 - frame.message?.content?.[0]?.id === OPEN_TOOL_ID
635 - ) {
636 - return [
637 - {
638 - ...entry,
639 - frame: {
640 - ...frame,
641 - message: {
642 - ...frame.message,
643 - content: [
644 - {
645 - ...frame.message.content[0],
646 - name: tool,
647 - input:
648 - tool === "WebFetch"
649 - ? { url: "https://example.com", prompt: "Summarize" }
650 - : { file_path: "notes.txt", content: "SUB_DONE_2" },
651 - },
652 - ],
653 - },
654 - },
655 - },
656 - ];
657 - }
658 - if (
659 - (frame.type === "system" &&
660 - (frame.subtype === "task_started" || frame.subtype === "task_notification") &&
661 - frame.tool_use_id === OPEN_TOOL_ID) ||
662 - (frame.type === "user" &&
663 - frame.message?.content?.[0]?.tool_use_id === OPEN_TOOL_ID) ||
664 - (frame.type === "assistant" &&
665 - frame.parent_tool_use_id !== null &&
666 - frame.message?.content?.[0]?.text === "SUB_FINAL_REPORT")
667 - ) {
668 - return [];
669 - }
670 - if (frame.type === "system" && frame.subtype === "task_updated") {
671 - return [
672 - {
673 - ...entry,
674 - frame: {
675 - ...frame,
676 - patch: {
677 - ...frame.patch,
678 - status: notificationStatus === "stopped" ? "killed" : "failed",
679 - },
680 - },
681 - },
682 - ];
683 - }
684 - if (
685 - frame.type === "system" &&
686 - frame.subtype === "task_notification" &&
687 - frame.task_id === SUBAGENT_TASK_ID
688 - ) {
689 - return [{ ...entry, frame: { ...frame, status: notificationStatus } }];
690 - }
691 - return [entry];
692 - }),
693 - ),
694 - }),
695 - }),
696 - );
697 - }
698 -});
699 -
700 -function undecodableEvent(entry: ProviderReplayEntry): ProviderReplayEntry {
701 - if (entry.type !== "emit_inbound") return entry;
702 - const frame = entry.frame as { readonly event: Record<string, unknown> };
703 - return { ...entry, frame: { ...frame, event: { ...frame.event, durable: "not-an-envelope" } } };
704 -}
apps/server/src/orchestration-v2/testkit/PiReplayFixtures.integration.test.ts new
+10
@@ -0,0 +1,10 @@
1 +import { describe } from "@effect/vitest";
2 +
3 +import { PiOrchestratorReplayHarness } from "../Adapters/PiAdapterV2.testkit.ts";
4 +import { itRunsReplayFixtures, runReplayFixture } from "./ProviderReplayFixtures.ts";
5 +
6 +describe("Pi replay fixtures", () => {
7 + itRunsReplayFixtures("pi", (input) =>
8 + runReplayFixture({ ...input, harness: PiOrchestratorReplayHarness }),
9 + );
10 +});
apps/server/src/orchestration-v2/testkit/ProviderReplayFixtures.ts new
+219
@@ -0,0 +1,219 @@
1 +import * as NodeServices from "@effect/platform-node/NodeServices";
2 +import { assert, it } from "@effect/vitest";
3 +import type { OrchestrationV2DomainEvent, ProviderReplayTranscript } from "@t3tools/contracts";
4 +import * as Effect from "effect/Effect";
5 +import * as FileSystem from "effect/FileSystem";
6 +import * as Path from "effect/Path";
7 +
8 +import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
9 +import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace";
10 +import { readProviderReplayTranscript } from "@t3tools/provider-testing/replayTranscript";
11 +import { provideDeterministicTestRuntime } from "./DeterministicRuntime.ts";
12 +import { ORCHESTRATOR_REPLAY_FIXTURES } from "./fixtures/index.ts";
13 +import {
14 + assertProviderNativeSubagentRootTurns,
15 + materializeFixtureInput,
16 + type OrchestratorFixtureInput,
17 + type ProviderOrchestratorReplayVariant,
18 +} from "./fixtures/shared.ts";
19 +import {
20 + runOrchestratorV2ProviderReplayScenario,
21 + type OrchestratorV2ProviderReplayHarness,
22 +} from "./ProviderReplayHarness.ts";
23 +import { materializeReplayTranscriptRuntimeInstructions } from "./ReplayRuntimeInstructions.ts";
24 +
25 +// Each driver's fixtures run from its own <Provider>ReplayFixtures.integration.test.ts,
26 +// so a focused run loads one adapter. A fixture for a driver missing here would
27 +// silently never run, so loading this module fails instead.
28 +const DRIVERS_WITH_REPLAY_FIXTURE_FILES = new Set([
29 + "acpRegistry",
30 + "claudeAgent",
31 + "codex",
32 + "cursor",
33 + "grok",
34 + "muse",
35 + "opencode",
36 + "pi",
37 +]);
38 +for (const fixture of ORCHESTRATOR_REPLAY_FIXTURES) {
39 + for (const provider of fixture.providers) {
40 + if (!DRIVERS_WITH_REPLAY_FIXTURE_FILES.has(provider.driver)) {
41 + throw new Error(
42 + `${fixture.name}/${provider.driver} has no ReplayFixtures test file to run it.`,
43 + );
44 + }
45 + }
46 +}
47 +
48 +export const readReplayFixtureTranscript = Effect.fn("readOrchestratorReplayTranscript")(function* (
49 + file: URL,
50 +) {
51 + return yield* readProviderReplayTranscript(file);
52 +}, Effect.provide(NodeServices.layer));
53 +
54 +function normalizeTestError(cause: unknown): Error {
55 + return cause instanceof Error ? cause : new Error(String(cause));
56 +}
57 +
58 +function transcriptEntriesThroughLabel(
59 + transcript: ProviderReplayTranscript,
60 + label: string | undefined,
61 +): ProviderReplayTranscript {
62 + if (label === undefined) {
63 + return transcript;
64 + }
65 + const entryIndex = transcript.entries.findIndex(
66 + (entry) => entry.type === "emit_inbound" && entry.label === label,
67 + );
68 + if (entryIndex === -1) {
69 + throw new Error(`${transcript.scenario} is missing inbound transcript label ${label}.`);
70 + }
71 + return {
72 + ...transcript,
73 + entries: transcript.entries.slice(0, entryIndex + 1),
74 + };
75 +}
76 +
77 +function isStreamingAssistantEvent(event: OrchestrationV2DomainEvent): boolean {
78 + switch (event.type) {
79 + case "node.updated":
80 + return event.payload.kind === "assistant_message" && event.payload.status === "running";
81 + case "message.updated":
82 + return event.payload.role === "assistant" && event.payload.streaming;
83 + case "turn-item.updated":
84 + return event.payload.type === "assistant_message" && event.payload.streaming;
85 + default:
86 + return false;
87 + }
88 +}
89 +
90 +export interface ReplayFixtureCase {
91 + readonly fixtureName: string;
92 + readonly buildInput: () => OrchestratorFixtureInput;
93 + readonly driver: ProviderOrchestratorReplayVariant;
94 + readonly transformTranscript?: (transcript: ProviderReplayTranscript) => ProviderReplayTranscript;
95 +}
96 +
97 +export interface ReplayFixtureRun<
98 + Transcript extends ProviderReplayTranscript,
99 + Error,
100 + WorkspaceError = never,
101 +> extends ReplayFixtureCase {
102 + readonly harness: OrchestratorV2ProviderReplayHarness<Transcript, Error>;
103 + /** Rewrites recorded workspace paths to the replay workspace, for drivers that record them. */
104 + readonly materializeWorkspace?: (
105 + transcript: ProviderReplayTranscript,
106 + workspace: string,
107 + ) => Effect.Effect<ProviderReplayTranscript, WorkspaceError>;
108 +}
109 +
110 +const runFixtureProvider = Effect.fn("runOrchestratorReplayFixture")(function* <
111 + Transcript extends ProviderReplayTranscript,
112 + Error,
113 + WorkspaceError,
114 +>(input: ReplayFixtureRun<Transcript, Error, WorkspaceError>) {
115 + const recordedTranscript = yield* readReplayFixtureTranscript(input.driver.transcriptFile);
116 + const rawTranscript = input.transformTranscript?.(recordedTranscript) ?? recordedTranscript;
117 + const replayTranscript = materializeReplayTranscriptRuntimeInstructions(
118 + transcriptEntriesThroughLabel(rawTranscript, input.driver.transcriptEntriesThroughLabel),
119 + { driver: input.driver.driver, model: input.driver.modelSelection.model },
120 + );
121 + const fixtureInput = input.buildInput();
122 + const workspace = yield* checkpointWorkspace(input.fixtureName, fixtureInput.workspaceFiles);
123 + const transcript = yield* input.harness.decodeTranscript(
124 + input.materializeWorkspace === undefined
125 + ? replayTranscript
126 + : yield* input.materializeWorkspace(replayTranscript, workspace),
127 + );
128 + const materialized = yield* materializeFixtureInput({
129 + scenario: input.fixtureName,
130 + fixtureInput,
131 + driver: input.driver.driver,
132 + modelSelection: input.driver.modelSelection,
133 + }).pipe(Effect.provide(IdAllocator.layer), provideDeterministicTestRuntime);
134 + const scenario = {
135 + name: `${input.fixtureName}/${input.driver.driver}`,
136 + transcript,
137 + commands: materialized.commands,
138 + steps: materialized.steps,
139 + projectionThreadIds: materialized.projectionThreadIds,
140 + runtimePolicyOverride: {
141 + ...input.driver.runtimePolicyOverride,
142 + cwd: workspace,
143 + },
144 + };
145 +
146 + const result = yield* runOrchestratorV2ProviderReplayScenario(
147 + scenario,
148 + input.harness,
149 + input.driver.runContinuationWorker === true ? { runContinuationWorker: true } : {},
150 + ).pipe(provideDeterministicTestRuntime);
151 + input.driver.assertOutput(result, transcript);
152 + assertProviderNativeSubagentRootTurns(result);
153 + const expectedAbsentWorkspacePaths = input.driver.expectedAbsentWorkspacePaths;
154 + if (expectedAbsentWorkspacePaths !== undefined) {
155 + yield* Effect.gen(function* () {
156 + const fs = yield* FileSystem.FileSystem;
157 + const path = yield* Path.Path;
158 + for (const relativePath of expectedAbsentWorkspacePaths) {
159 + assert.isFalse(
160 + yield* fs.exists(path.join(workspace, relativePath)),
161 + `${input.fixtureName}/${input.driver.driver} must not create ${relativePath} in the replay workspace`,
162 + );
163 + }
164 + }).pipe(Effect.provide(NodeServices.layer));
165 + }
166 + assert.isFalse(
167 + result.domainEvents.some(isStreamingAssistantEvent),
168 + "buffered delivery must not persist streaming assistant artifacts",
169 + );
170 + const projectionThreadId = materialized.projectionThreadIds[0];
171 + assert.isDefined(projectionThreadId);
172 + const projection = result.projections.get(projectionThreadId);
173 + assert.isDefined(projection);
174 + const latestRun = projection.runs.at(-1);
175 + assert.deepEqual(latestRun?.modelSelection, input.driver.modelSelection);
176 + if (projection.runs.some((run) => run.status === "completed")) {
177 + const threadStartCheckpoint = projection.checkpoints.find(
178 + (checkpoint) => checkpoint.ordinalWithinScope === 0 && checkpoint.appRunOrdinal === null,
179 + );
180 + assert.isDefined(
181 + threadStartCheckpoint,
182 + "completed threads must retain an addressable thread-start checkpoint",
183 + );
184 + assert.equal(threadStartCheckpoint.status, "ready");
185 + }
186 + return result;
187 +});
188 +
189 +/** Replays one fixture variant through OrchestratorV2 and runs the shared output checks. */
190 +export function runReplayFixture<
191 + Transcript extends ProviderReplayTranscript,
192 + Error,
193 + WorkspaceError = never,
194 +>(input: ReplayFixtureRun<Transcript, Error, WorkspaceError>) {
195 + return runFixtureProvider(input).pipe(Effect.mapError(normalizeTestError), Effect.scoped);
196 +}
197 +
198 +export function findReplayFixture(name: string) {
199 + return ORCHESTRATOR_REPLAY_FIXTURES.find((fixture) => fixture.name === name);
200 +}
201 +
202 +/**
203 + * Registers one test per recorded fixture variant of `driver`. Call it inside the
204 + * provider file's `describe`. `run` picks the harness for each variant.
205 + */
206 +export function itRunsReplayFixtures<E>(
207 + driver: string,
208 + run: (input: ReplayFixtureCase) => Effect.Effect<unknown, E>,
209 +) {
210 + it.effect.each(
211 + ORCHESTRATOR_REPLAY_FIXTURES.flatMap((fixture) =>
212 + fixture.providers
213 + .filter((provider) => provider.driver === driver)
214 + .map((provider) => [fixture.name, provider.driver, fixture, provider] as const),
215 + ),
216 + )("runs %s/%s through OrchestratorV2 using deterministic replay", ([, , fixture, provider]) =>
217 + run({ fixtureName: fixture.name, buildInput: fixture.buildInput, driver: provider }),
218 + );
219 +}
apps/server/src/testUtils/shardWeights.json
+8 -2
@@ -123,10 +123,16 @@
123 "src/orchestration-v2/legacy/LegacyV1Cutover.integration.test.ts": 1.4,
124 "src/orchestration-v2/legacy/LegacyV1ThreadImporter.test.ts": 0.5,
125 "src/orchestration-v2/runtimeLayer.test.ts": 2.8,
126 - "src/orchestration-v2/testkit/ClaudeReplayFixtures.integration.test.ts": 2.7,
127 - "src/orchestration-v2/testkit/OrchestratorReplayFixtures.integration.test.ts": 44.6,
126 + "src/orchestration-v2/testkit/AcpRegistryReplayFixtures.integration.test.ts": 4.8,
127 + "src/orchestration-v2/testkit/ClaudeReplayFixtures.integration.test.ts": 13.7,
128 + "src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts": 6.4,
129 + "src/orchestration-v2/testkit/CursorReplayFixtures.integration.test.ts": 4.8,
130 + "src/orchestration-v2/testkit/GrokReplayFixtures.integration.test.ts": 5.1,
131 + "src/orchestration-v2/testkit/MuseReplayFixtures.integration.test.ts": 2.4,
132 + "src/orchestration-v2/testkit/OpenCodeReplayFixtures.integration.test.ts": 5.8,
133 "src/orchestration-v2/testkit/OrchestratorReplayRecovery.integration.test.ts": 2.2,
134 "src/orchestration-v2/testkit/OrchestratorReplayRestartBackgroundNote.integration.test.ts": 2.1,
135 + "src/orchestration-v2/testkit/PiReplayFixtures.integration.test.ts": 4.3,
136 "src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts": 16.1,
137 "src/orchestration-v2/testkit/ThreadFork.integration.test.ts": 2.9,
138 "src/orchestration-v2/testkit/ThreadMergeBack.integration.test.ts": 5.1,