The best way to control Claude Code, Codex and more from any computer or phone
test(server): split replay fixtures into per-provider files (#22717)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Theo Browne committed
Oct 21, 2026 at 00:18 UTC
dd8452a06599732784c8f1725fde5e8695d652f2
11 files changed
+817
-707
apps/server/src/orchestration-v2/testkit/AcpRegistryReplayFixtures.integration.test.ts
new
+213
@@ -0,0 +1,213 @@
1
+import * as NodeServices from "@effect/platform-node/NodeServices";
2
+import { assert, describe, it } from "@effect/vitest";
3
+import {
4
+ ProviderDriverKind,
5
+ ProviderInstanceId,
6
+ type ProviderReplayTranscript,
7
+} from "@t3tools/contracts";
8
+import * as DateTime from "effect/DateTime";
9
+import * as Effect from "effect/Effect";
10
+import * as Layer from "effect/Layer";
11
+import * as Option from "effect/Option";
12
+import * as Stream from "effect/Stream";
13
+import { TestClock } from "effect/testing";
14
+
15
+import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
16
+import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace";
17
+import { AcpRegistryOrchestratorReplayHarness } from "../Adapters/AcpRegistryAdapterV2.testkit.ts";
18
+import * as EffectWorker from "../EffectWorker.ts";
19
+import * as Orchestrator from "../Orchestrator.ts";
20
+import { provideDeterministicTestRuntime } from "./DeterministicRuntime.ts";
21
+import { materializeFixtureInput, SIMPLE_PROMPT } from "./fixtures/shared.ts";
22
+import {
23
+ itRunsReplayFixtures,
24
+ readReplayFixtureTranscript,
25
+ runReplayFixture,
26
+} from "./ProviderReplayFixtures.ts";
27
+import { layerProviderReplay } from "./ProviderReplayHarness.ts";
28
+import { materializeReplayTranscriptRuntimeInstructions } from "./ReplayRuntimeInstructions.ts";
29
+
30
+describe("ACP registry replay fixtures", () => {
31
+ it.effect.each([
32
+ { name: "open", firstTurn: false, code: -32602 },
33
+ { name: "switch", firstTurn: true, code: -32602 },
34
+ { name: "authentication", firstTurn: false, code: -32000 },
35
+ ])("preserves the ACP model-write $name failure through orchestration", (testCase) =>
36
+ Effect.gen(function* () {
37
+ const name = `acp_model_write_${testCase.name}`;
38
+ const sessionId = "grok-replay-session-1";
39
+ const rejectedModel = "replay-model-b";
40
+ const modelSelection = {
41
+ instanceId: ProviderInstanceId.make("acpRegistry"),
42
+ model: rejectedModel,
43
+ };
44
+ const recorded = yield* readReplayFixtureTranscript(
45
+ new URL("./fixtures/simple/registry_transcript.ndjson", import.meta.url),
46
+ );
47
+ const sessionIndex = recorded.entries.findIndex(
48
+ (entry) => "label" in entry && entry.label === "session.created",
49
+ );
50
+ assert.isAtLeast(sessionIndex, 0);
51
+ const transcript = {
52
+ ...recorded,
53
+ scenario: name,
54
+ entries: [
55
+ ...recorded.entries.slice(0, sessionIndex),
56
+ {
57
+ type: "emit_inbound",
58
+ label: "session.created",
59
+ frame: {
60
+ kind: "response",
61
+ method: "session/new",
62
+ result: {
63
+ sessionId,
64
+ configOptions: [
65
+ {
66
+ id: "model",
67
+ name: "Model",
68
+ category: "model",
69
+ type: "select",
70
+ currentValue: "replay-model-a",
71
+ options: [
72
+ { value: "replay-model-a", name: "Model A" },
73
+ { value: rejectedModel, name: "Model B" },
74
+ ],
75
+ },
76
+ ],
77
+ },
78
+ },
79
+ },
80
+ ...(testCase.firstTurn
81
+ ? recorded.entries
82
+ .slice(sessionIndex + 1)
83
+ .filter((entry) => entry.type !== "runtime_exit")
84
+ : []),
85
+ {
86
+ type: "expect_outbound",
87
+ label: "session.set_model",
88
+ frame: {
89
+ kind: "request",
90
+ method: "session/set_config_option",
91
+ params: { sessionId, configId: "model", value: rejectedModel },
92
+ },
93
+ },
94
+ {
95
+ type: "emit_inbound",
96
+ label: "session.set_model.rejected",
97
+ frame: {
98
+ kind: "response",
99
+ method: "session/set_config_option",
100
+ error: {
101
+ code: testCase.code,
102
+ message: testCase.code === -32602 ? "Invalid provider" : "Authentication required",
103
+ },
104
+ },
105
+ },
106
+ ],
107
+ } satisfies ProviderReplayTranscript;
108
+ const materialized = yield* materializeFixtureInput({
109
+ scenario: name,
110
+ fixtureInput: {
111
+ steps: Array.from({ length: testCase.firstTurn ? 2 : 1 }, () => ({
112
+ type: "message" as const,
113
+ text: SIMPLE_PROMPT,
114
+ })),
115
+ },
116
+ driver: ProviderDriverKind.make("acpRegistry"),
117
+ modelSelection: {
118
+ ...modelSelection,
119
+ model: testCase.firstTurn ? "replay-model-a" : rejectedModel,
120
+ },
121
+ });
122
+ const scenario = {
123
+ ...materialized,
124
+ name,
125
+ transcript: yield* AcpRegistryOrchestratorReplayHarness.decodeTranscript(
126
+ materializeReplayTranscriptRuntimeInstructions(transcript, {
127
+ driver: ProviderDriverKind.make("acpRegistry"),
128
+ model: "replay-model-a",
129
+ }),
130
+ ),
131
+ runtimePolicyOverride: { cwd: yield* checkpointWorkspace(name) },
132
+ };
133
+ yield* Effect.gen(function* () {
134
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
135
+ const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
136
+ if (testCase.firstTurn) yield* EffectWorker.runDaemon.pipe(Effect.forkScoped);
137
+ for (const [index, command] of materialized.commands.entries()) {
138
+ const isFirstTurn =
139
+ testCase.firstTurn && command.type === "message.dispatch" && index === 1;
140
+ yield* orchestrator.dispatch(
141
+ command.type === "message.dispatch" && !isFirstTurn
142
+ ? { ...command, modelSelection }
143
+ : command,
144
+ );
145
+ if (isFirstTurn) {
146
+ // Wait for the committed terminal event before switching models.
147
+ yield* orchestrator.streamStoredEvents.pipe(
148
+ Stream.filter(
149
+ ({ event }) => event.type === "run.updated" && event.payload.status === "completed",
150
+ ),
151
+ Stream.runHead,
152
+ );
153
+ }
154
+ }
155
+ if (testCase.firstTurn) {
156
+ yield* orchestrator.streamStoredEvents.pipe(
157
+ Stream.filter(
158
+ ({ event }) => event.type === "run.updated" && event.payload.status === "failed",
159
+ ),
160
+ Stream.runHead,
161
+ );
162
+ } else {
163
+ // Failed opens retry on the outbox schedule. Advance to each recorded
164
+ // deadline, draining effects instead of polling the projection.
165
+ for (let attempt = 0; attempt < 10; attempt++) {
166
+ yield* worker.drain();
167
+ const next = yield* worker.nextClaimableAt;
168
+ if (Option.isNone(next)) break;
169
+ const now = yield* DateTime.now;
170
+ yield* TestClock.adjust(
171
+ Math.max(0, DateTime.toEpochMillis(next.value) - DateTime.toEpochMillis(now)),
172
+ );
173
+ }
174
+ }
175
+ const projection = yield* orchestrator.getThreadProjection(
176
+ materialized.projectionThreadIds[0]!,
177
+ );
178
+ assert.deepEqual(
179
+ projection.runs.map((run) => run.status),
180
+ testCase.firstTurn ? ["completed", "failed"] : ["failed"],
181
+ );
182
+ const errors = projection.turnItems.filter((item) => item.type === "error");
183
+ assert.lengthOf(errors, 1);
184
+ assert.equal(errors[0]?.failure.class, "provider_error");
185
+ if (testCase.code === -32602) {
186
+ assert.equal(
187
+ errors[0]?.failure.message,
188
+ 'The agent did not accept the model "replay-model-b". Choose another model for this provider, then retry.',
189
+ );
190
+ } else {
191
+ assert.match(
192
+ errors[0]?.failure.message ?? "",
193
+ /^Failed to open acpRegistry provider session \S+\.$/u,
194
+ );
195
+ }
196
+ }).pipe(
197
+ Effect.provide(
198
+ layerProviderReplay(scenario, AcpRegistryOrchestratorReplayHarness, {
199
+ runEffectWorker: false,
200
+ }),
201
+ ),
202
+ );
203
+ }).pipe(
204
+ Effect.scoped,
205
+ provideDeterministicTestRuntime,
206
+ Effect.provide(Layer.merge(IdAllocator.layer, NodeServices.layer)),
207
+ ),
208
+ );
209
+
210
+ itRunsReplayFixtures("acpRegistry", (input) =>
211
+ runReplayFixture({ ...input, harness: AcpRegistryOrchestratorReplayHarness }),
212
+ );
213
+});
apps/server/src/orchestration-v2/testkit/ClaudeReplayFixtures.integration.test.ts
+226
-1
@@ -5,6 +5,7 @@ import {
5
isProviderNativeSubagentThread,
6
MessageId,
7
ProviderDriverKind,
8
+ type ProviderReplayEntry,
9
} from "@t3tools/contracts";
10
import * as Effect from "effect/Effect";
11
@@ -17,7 +18,17 @@ import { subagentInput } from "./fixtures/subagent/input.ts";
18
import { runOrchestratorV2Scenario } from "./OrchestratorScenario.ts";
19
import * as ProviderReplayHarness from "./ProviderReplayHarness.ts";
20
import { materializeReplayTranscriptRuntimeInstructions } from "./ReplayRuntimeInstructions.ts";
20
-import { CLAUDE_MODEL_SELECTION, materializeFixtureInput } from "./fixtures/shared.ts";
21
+import {
22
+ CLAUDE_MODEL_SELECTION,
23
+ materializeFixtureInput,
24
+ projectionFor,
25
+} from "./fixtures/shared.ts";
26
+import {
27
+ findReplayFixture,
28
+ itRunsReplayFixtures,
29
+ runReplayFixture,
30
+ type ReplayFixtureCase,
31
+} from "./ProviderReplayFixtures.ts";
32
import { readProviderReplayTranscript } from "@t3tools/provider-testing/replayTranscript";
33
34
const readTranscript = Effect.fn("readClaudeReplayFixture")(function* (file: URL) {
@@ -28,7 +39,13 @@ function readClaudeTranscriptFixture(path: string) {
39
return readTranscript(new URL(`./fixtures/${path}/claude_transcript.ndjson`, import.meta.url));
40
}
41
42
+function runClaudeReplayFixture(input: ReplayFixtureCase) {
43
+ return runReplayFixture({ ...input, harness: ClaudeOrchestratorReplayHarness });
44
+}
45
+
46
describe("Claude Agent SDK replay fixtures", () => {
47
+ itRunsReplayFixtures("claudeAgent", runClaudeReplayFixture);
48
+
49
it.effect("refuses messages to a native subagent thread without touching it", () =>
50
Effect.gen(function* () {
51
const raw = yield* readClaudeTranscriptFixture("subagent");
@@ -91,4 +108,212 @@ describe("Claude Agent SDK replay fixtures", () => {
108
);
109
}),
110
);
111
+
112
+ const afterRootFixture = findReplayFixture("claude_background_subagent_after_root");
113
+ const afterRootProvider = afterRootFixture?.providers[0];
114
+ if (afterRootFixture !== undefined && afterRootProvider !== undefined) {
115
+ const OPEN_TOOL_ID = "toolu_01QDa5jV5g1H9h6QyDfeJohD";
116
+ const SUBAGENT_TASK_ID = "a2995bfced8019363";
117
+
118
+ const moveOpenToolBeforeRootResult = (
119
+ move: boolean,
120
+ entries: ReadonlyArray<ProviderReplayEntry>,
121
+ ): ReadonlyArray<ProviderReplayEntry> => {
122
+ if (!move) return entries;
123
+ const toolIndex = entries.findIndex(
124
+ (entry) =>
125
+ entry.type === "emit_inbound" &&
126
+ (entry.frame as Record<string, any>).message?.content?.[0]?.id === OPEN_TOOL_ID,
127
+ );
128
+ const resultIndex = entries.findIndex(
129
+ (entry) => entry.type === "emit_inbound" && entry.label === "result",
130
+ );
131
+ if (toolIndex < 0 || resultIndex < 0) throw new Error("transcript shape changed");
132
+ const toolEntry = entries[toolIndex]!;
133
+ const rest = entries.filter((_, index) => index !== toolIndex);
134
+ return [...rest.slice(0, resultIndex), toolEntry, ...rest.slice(resultIndex)];
135
+ };
136
+ // A subagent's foreground Bash run during the root's turn is tailed like a
137
+ // root call's, so the child's command row streams live.
138
+ const SUBAGENT_BASH_ID = "toolu_01JMN3SSQCHzUTNua9AWdkiH";
139
+ it.effect("streams a subagent's foreground Bash output live during the root turn", () =>
140
+ runClaudeReplayFixture({
141
+ fixtureName: afterRootFixture.name,
142
+ buildInput: afterRootFixture.buildInput,
143
+ driver: {
144
+ ...afterRootProvider,
145
+ assertOutput: (result) => {
146
+ const childThreadId = projectionFor(result, afterRootFixture.name).subagents[0]
147
+ ?.childThreadId;
148
+ assert.exists(childThreadId);
149
+ const command = result.projections
150
+ .get(childThreadId)
151
+ ?.turnItems.find(
152
+ (item) =>
153
+ item.type === "command_execution" &&
154
+ JSON.stringify(item.input).includes("SUB_DONE_1"),
155
+ );
156
+ assert.exists(command);
157
+ const streamed = result.commandOutput?.get(command.id);
158
+ assert.exists(streamed, "the subagent's command must stream its output");
159
+ assert.include(streamed.output.text, "SUB_DONE_1");
160
+ },
161
+ },
162
+ transformTranscript: (transcript) => {
163
+ const isBashFrame = (entry: ProviderReplayEntry) => {
164
+ if (entry.type !== "emit_inbound") return false;
165
+ const frame = entry.frame as Record<string, any>;
166
+ return (
167
+ frame.message?.content?.[0]?.id === SUBAGENT_BASH_ID ||
168
+ frame.message?.content?.[0]?.tool_use_id === SUBAGENT_BASH_ID ||
169
+ (frame.type === "system" && frame.tool_use_id === SUBAGENT_BASH_ID)
170
+ );
171
+ };
172
+ // Recorded without the structured result the replay writes output files from.
173
+ const bashFrames = transcript.entries.filter(isBashFrame).map((entry) => {
174
+ const frame = (entry as { readonly frame: Record<string, any> }).frame;
175
+ return frame.type === "user"
176
+ ? {
177
+ ...entry,
178
+ frame: { ...frame, tool_use_result: { stdout: "SUB_DONE_1\n", stderr: "" } },
179
+ }
180
+ : entry;
181
+ });
182
+ const rest = transcript.entries.filter((entry) => !isBashFrame(entry));
183
+ const resultIndex = rest.findIndex(
184
+ (entry) => entry.type === "emit_inbound" && entry.label === "result",
185
+ );
186
+ assert.lengthOf(bashFrames, 4, "transcript shape changed");
187
+ assert.isAtLeast(resultIndex, 0, "transcript shape changed");
188
+ return {
189
+ ...transcript,
190
+ entries: [...rest.slice(0, resultIndex), ...bashFrames, ...rest.slice(resultIndex)],
191
+ };
192
+ },
193
+ }),
194
+ );
195
+
196
+ // regression: #16486 a background subagent's tool left open while its parent is idle never settled
197
+ it.effect.each([
198
+ ["WebFetch", "while root is idle", "stopped"],
199
+ ["Write", "before root settles", "failed"],
200
+ ] as const)(
201
+ "ends a background subagent's open %s (opened %s) when its notification is %s",
202
+ ([tool, opens, notificationStatus]) =>
203
+ runClaudeReplayFixture({
204
+ fixtureName: afterRootFixture.name,
205
+ buildInput: afterRootFixture.buildInput,
206
+ driver: {
207
+ ...afterRootProvider,
208
+ assertOutput: (result) => {
209
+ const parent = projectionFor(result, afterRootFixture.name);
210
+ assert.deepEqual(
211
+ parent.runs.map((run) => run.status),
212
+ ["completed", "completed"],
213
+ );
214
+ const subagent = parent.subagents[0];
215
+ assert.equal(
216
+ subagent?.status,
217
+ notificationStatus === "stopped" ? "cancelled" : "failed",
218
+ );
219
+ const childThreadId = subagent?.childThreadId;
220
+ assert.exists(childThreadId);
221
+ const child = result.projections.get(childThreadId);
222
+ assert.exists(child);
223
+ const fetches = child.turnItems.filter(
224
+ (item) => item.type === (tool === "WebFetch" ? "web_search" : "file_change"),
225
+ );
226
+ assert.lengthOf(fetches, 1, `the ${tool} reached the child thread`);
227
+ assert.deepEqual(
228
+ {
229
+ openTool: fetches.map((item) => item.status),
230
+ openChildNodes: child.nodes
231
+ .filter(
232
+ (node) =>
233
+ node.status === "running" ||
234
+ node.status === "pending" ||
235
+ node.status === "waiting",
236
+ )
237
+ .map((node) => `${node.kind}:${node.status}`),
238
+ },
239
+ {
240
+ openTool: [notificationStatus === "stopped" ? "interrupted" : "failed"],
241
+ openChildNodes: [],
242
+ },
243
+ );
244
+ },
245
+ },
246
+ transformTranscript: (transcript) => ({
247
+ ...transcript,
248
+ entries: moveOpenToolBeforeRootResult(
249
+ opens === "before root settles",
250
+ transcript.entries.flatMap((entry): ReadonlyArray<ProviderReplayEntry> => {
251
+ if (entry.type !== "emit_inbound") return [entry];
252
+ const frame = entry.frame as Record<string, any>;
253
+ if (
254
+ frame.type === "assistant" &&
255
+ frame.message?.content?.[0]?.id === OPEN_TOOL_ID
256
+ ) {
257
+ return [
258
+ {
259
+ ...entry,
260
+ frame: {
261
+ ...frame,
262
+ message: {
263
+ ...frame.message,
264
+ content: [
265
+ {
266
+ ...frame.message.content[0],
267
+ name: tool,
268
+ input:
269
+ tool === "WebFetch"
270
+ ? { url: "https://example.com", prompt: "Summarize" }
271
+ : { file_path: "notes.txt", content: "SUB_DONE_2" },
272
+ },
273
+ ],
274
+ },
275
+ },
276
+ },
277
+ ];
278
+ }
279
+ if (
280
+ (frame.type === "system" &&
281
+ (frame.subtype === "task_started" || frame.subtype === "task_notification") &&
282
+ frame.tool_use_id === OPEN_TOOL_ID) ||
283
+ (frame.type === "user" &&
284
+ frame.message?.content?.[0]?.tool_use_id === OPEN_TOOL_ID) ||
285
+ (frame.type === "assistant" &&
286
+ frame.parent_tool_use_id !== null &&
287
+ frame.message?.content?.[0]?.text === "SUB_FINAL_REPORT")
288
+ ) {
289
+ return [];
290
+ }
291
+ if (frame.type === "system" && frame.subtype === "task_updated") {
292
+ return [
293
+ {
294
+ ...entry,
295
+ frame: {
296
+ ...frame,
297
+ patch: {
298
+ ...frame.patch,
299
+ status: notificationStatus === "stopped" ? "killed" : "failed",
300
+ },
301
+ },
302
+ },
303
+ ];
304
+ }
305
+ if (
306
+ frame.type === "system" &&
307
+ frame.subtype === "task_notification" &&
308
+ frame.task_id === SUBAGENT_TASK_ID
309
+ ) {
310
+ return [{ ...entry, frame: { ...frame, status: notificationStatus } }];
311
+ }
312
+ return [entry];
313
+ }),
314
+ ),
315
+ }),
316
+ }),
317
+ );
318
+ }
319
});
apps/server/src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts
new
+17
@@ -0,0 +1,17 @@
1
+import { describe } from "@effect/vitest";
2
+import * as Effect from "effect/Effect";
3
+
4
+import { materializeReplayTranscriptWorkspace } from "@t3tools/provider-testing/replayTranscript";
5
+import { CodexOrchestratorReplayHarness } from "../Adapters/CodexAdapterV2.testkit.ts";
6
+import { itRunsReplayFixtures, runReplayFixture } from "./ProviderReplayFixtures.ts";
7
+
8
+describe("Codex replay fixtures", () => {
9
+ itRunsReplayFixtures("codex", (input) =>
10
+ runReplayFixture({
11
+ ...input,
12
+ harness: CodexOrchestratorReplayHarness,
13
+ materializeWorkspace: (transcript, workspace) =>
14
+ Effect.succeed(materializeReplayTranscriptWorkspace(transcript, workspace)),
15
+ }),
16
+ );
17
+});
apps/server/src/orchestration-v2/testkit/CursorReplayFixtures.integration.test.ts
new
+29
@@ -0,0 +1,29 @@
1
+import { describe, it } from "@effect/vitest";
2
+
3
+import { CursorOrchestratorReplayHarness } from "../Adapters/CursorAdapterV2.testkit.ts";
4
+import { messageRestartInput } from "./fixtures/message_steering/input.ts";
5
+import {
6
+ findReplayFixture,
7
+ itRunsReplayFixtures,
8
+ runReplayFixture,
9
+} from "./ProviderReplayFixtures.ts";
10
+
11
+describe("Cursor replay fixtures", () => {
12
+ itRunsReplayFixtures("cursor", (input) =>
13
+ runReplayFixture({ ...input, harness: CursorOrchestratorReplayHarness }),
14
+ );
15
+
16
+ const cursorSteeringProvider = findReplayFixture("message_steering")?.providers.find(
17
+ (provider) => provider.driver === "cursor",
18
+ );
19
+ if (cursorSteeringProvider !== undefined) {
20
+ it.effect("executes explicit Cursor restart_active through the recorded SDK boundary", () =>
21
+ runReplayFixture({
22
+ fixtureName: "message_steering",
23
+ buildInput: messageRestartInput,
24
+ driver: cursorSteeringProvider,
25
+ harness: CursorOrchestratorReplayHarness,
26
+ }),
27
+ );
28
+ }
29
+});
apps/server/src/orchestration-v2/testkit/GrokReplayFixtures.integration.test.ts
new
+10
@@ -0,0 +1,10 @@
1
+import { describe } from "@effect/vitest";
2
+
3
+import { GrokOrchestratorReplayHarness } from "../Adapters/GrokAdapterV2.testkit.ts";
4
+import { itRunsReplayFixtures, runReplayFixture } from "./ProviderReplayFixtures.ts";
5
+
6
+describe("Grok replay fixtures", () => {
7
+ itRunsReplayFixtures("grok", (input) =>
8
+ runReplayFixture({ ...input, harness: GrokOrchestratorReplayHarness }),
9
+ );
10
+});
apps/server/src/orchestration-v2/testkit/MuseReplayFixtures.integration.test.ts
new
+17
@@ -0,0 +1,17 @@
1
+import { describe } from "@effect/vitest";
2
+
3
+import {
4
+ materializeMuseReplayWorkspace,
5
+ MuseOrchestratorReplayHarness,
6
+} from "../Adapters/MuseAdapterV2.testkit.ts";
7
+import { itRunsReplayFixtures, runReplayFixture } from "./ProviderReplayFixtures.ts";
8
+
9
+describe("Muse replay fixtures", () => {
10
+ itRunsReplayFixtures("muse", (input) =>
11
+ runReplayFixture({
12
+ ...input,
13
+ harness: MuseOrchestratorReplayHarness,
14
+ materializeWorkspace: materializeMuseReplayWorkspace,
15
+ }),
16
+ );
17
+});
apps/server/src/orchestration-v2/testkit/OpenCodeReplayFixtures.integration.test.ts
new
+68
@@ -0,0 +1,68 @@
1
+import { describe, it } from "@effect/vitest";
2
+import type { ProviderReplayEntry } from "@t3tools/contracts";
3
+import * as Effect from "effect/Effect";
4
+
5
+import { OpenCodeOrchestratorReplayHarness } from "../Adapters/OpenCodeAdapterV2.testkit.ts";
6
+import {
7
+ OPENCODE2_HTTP_PROTOCOL,
8
+ OpenCode2OrchestratorReplayHarness,
9
+} from "../Adapters/OpenCode2AdapterV2.testkit.ts";
10
+import {
11
+ findReplayFixture,
12
+ itRunsReplayFixtures,
13
+ readReplayFixtureTranscript,
14
+ runReplayFixture,
15
+ type ReplayFixtureCase,
16
+} from "./ProviderReplayFixtures.ts";
17
+
18
+// OpenCode fixtures are recorded against either the v1 or the v2 HTTP protocol.
19
+function runOpenCodeReplayFixture(input: ReplayFixtureCase) {
20
+ return readReplayFixtureTranscript(input.driver.transcriptFile).pipe(
21
+ Effect.flatMap((transcript) =>
22
+ transcript.protocol === OPENCODE2_HTTP_PROTOCOL
23
+ ? runReplayFixture({ ...input, harness: OpenCode2OrchestratorReplayHarness })
24
+ : runReplayFixture({ ...input, harness: OpenCodeOrchestratorReplayHarness }),
25
+ ),
26
+ );
27
+}
28
+
29
+describe("OpenCode replay fixtures", () => {
30
+ itRunsReplayFixtures("opencode", runOpenCodeReplayFixture);
31
+
32
+ it.effect.each(
33
+ (
34
+ [
35
+ ["opencode2_subagent", "session.execution.started.2"],
36
+ ["opencode2_background", "session.execution.started.3"],
37
+ ] as const
38
+ ).flatMap(([fixtureName, label]) => {
39
+ const fixture = findReplayFixture(fixtureName);
40
+ const provider = fixture?.providers[0];
41
+ return fixture === undefined || provider === undefined
42
+ ? []
43
+ : [[fixtureName, label, fixture, provider] as const];
44
+ }),
45
+ )(
46
+ "runs %s when %s is an execution start this build cannot decode",
47
+ ([fixtureName, label, fixture, provider]) =>
48
+ runOpenCodeReplayFixture({
49
+ fixtureName,
50
+ buildInput: fixture.buildInput,
51
+ driver: provider,
52
+ transformTranscript: (transcript) => ({
53
+ ...transcript,
54
+ entries: transcript.entries.map((entry) =>
55
+ entry.type === "emit_inbound" && entry.label === label
56
+ ? undecodableEvent(entry)
57
+ : entry,
58
+ ),
59
+ }),
60
+ }),
61
+ );
62
+});
63
+
64
+function undecodableEvent(entry: ProviderReplayEntry): ProviderReplayEntry {
65
+ if (entry.type !== "emit_inbound") return entry;
66
+ const frame = entry.frame as { readonly event: Record<string, unknown> };
67
+ return { ...entry, frame: { ...frame, event: { ...frame.event, durable: "not-an-envelope" } } };
68
+}
apps/server/src/orchestration-v2/testkit/OrchestratorReplayFixtures.integration.test.ts
deleted
-704
@@ -1,704 +0,0 @@
1
-import * as NodeServices from "@effect/platform-node/NodeServices";
2
-import { assert, describe, it } from "@effect/vitest";
3
-import {
4
- ProviderDriverKind,
5
- ProviderInstanceId,
6
- type OrchestrationV2DomainEvent,
7
- type ProviderReplayEntry,
8
- type ProviderReplayTranscript,
9
-} from "@t3tools/contracts";
10
-import * as DateTime from "effect/DateTime";
11
-import * as Effect from "effect/Effect";
12
-import * as FileSystem from "effect/FileSystem";
13
-import * as Layer from "effect/Layer";
14
-import * as Option from "effect/Option";
15
-import * as Path from "effect/Path";
16
-import * as Stream from "effect/Stream";
17
-import { TestClock } from "effect/testing";
18
-
19
-import * as EffectWorker from "../EffectWorker.ts";
20
-import * as Orchestrator from "../Orchestrator.ts";
21
-
22
-import { ClaudeOrchestratorReplayHarness } from "../Adapters/ClaudeAdapterV2.testkit.ts";
23
-import { CodexOrchestratorReplayHarness } from "../Adapters/CodexAdapterV2.testkit.ts";
24
-import { CursorOrchestratorReplayHarness } from "../Adapters/CursorAdapterV2.testkit.ts";
25
-import { AcpRegistryOrchestratorReplayHarness } from "../Adapters/AcpRegistryAdapterV2.testkit.ts";
26
-import { GrokOrchestratorReplayHarness } from "../Adapters/GrokAdapterV2.testkit.ts";
27
-import { OpenCodeOrchestratorReplayHarness } from "../Adapters/OpenCodeAdapterV2.testkit.ts";
28
-import {
29
- OPENCODE2_HTTP_PROTOCOL,
30
- OpenCode2OrchestratorReplayHarness,
31
-} from "../Adapters/OpenCode2AdapterV2.testkit.ts";
32
-import {
33
- materializeMuseReplayWorkspace,
34
- MuseOrchestratorReplayHarness,
35
-} from "../Adapters/MuseAdapterV2.testkit.ts";
36
-import { PiOrchestratorReplayHarness } from "../Adapters/PiAdapterV2.testkit.ts";
37
-import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
38
-import { provideDeterministicTestRuntime } from "./DeterministicRuntime.ts";
39
-import { ORCHESTRATOR_REPLAY_FIXTURES } from "./fixtures/index.ts";
40
-import { messageRestartInput } from "./fixtures/message_steering/input.ts";
41
-import {
42
- assertProviderNativeSubagentRootTurns,
43
- materializeFixtureInput,
44
- SIMPLE_PROMPT,
45
- projectionFor,
46
- type OrchestratorFixtureInput,
47
- type ProviderOrchestratorReplayVariant,
48
-} from "./fixtures/shared.ts";
49
-import {
50
- runOrchestratorV2ProviderReplayScenario,
51
- layerProviderReplay,
52
- type OrchestratorV2ProviderReplayHarness,
53
-} from "./ProviderReplayHarness.ts";
54
-import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace";
55
-import { materializeReplayTranscriptRuntimeInstructions } from "./ReplayRuntimeInstructions.ts";
56
-import {
57
- materializeReplayTranscriptWorkspace,
58
- readProviderReplayTranscript,
59
-} from "@t3tools/provider-testing/replayTranscript";
60
-
61
-const readTranscript = Effect.fn("readOrchestratorReplayTranscript")(function* (file: URL) {
62
- return yield* readProviderReplayTranscript(file);
63
-}, Effect.provide(NodeServices.layer));
64
-
65
-function normalizeTestError(cause: unknown): Error {
66
- return cause instanceof Error ? cause : new Error(String(cause));
67
-}
68
-
69
-function transcriptEntriesThroughLabel(
70
- transcript: ProviderReplayTranscript,
71
- label: string | undefined,
72
-): ProviderReplayTranscript {
73
- if (label === undefined) {
74
- return transcript;
75
- }
76
- const entryIndex = transcript.entries.findIndex(
77
- (entry) => entry.type === "emit_inbound" && entry.label === label,
78
- );
79
- if (entryIndex === -1) {
80
- throw new Error(`${transcript.scenario} is missing inbound transcript label ${label}.`);
81
- }
82
- return {
83
- ...transcript,
84
- entries: transcript.entries.slice(0, entryIndex + 1),
85
- };
86
-}
87
-
88
-function isStreamingAssistantEvent(event: OrchestrationV2DomainEvent): boolean {
89
- switch (event.type) {
90
- case "node.updated":
91
- return event.payload.kind === "assistant_message" && event.payload.status === "running";
92
- case "message.updated":
93
- return event.payload.role === "assistant" && event.payload.streaming;
94
- case "turn-item.updated":
95
- return event.payload.type === "assistant_message" && event.payload.streaming;
96
- default:
97
- return false;
98
- }
99
-}
100
-
101
-const runFixtureProvider = Effect.fn("runOrchestratorReplayFixture")(function* <
102
- Transcript extends ProviderReplayTranscript,
103
- Error,
104
->(input: {
105
- readonly fixtureName: string;
106
- readonly buildInput: () => OrchestratorFixtureInput;
107
- readonly driver: ProviderOrchestratorReplayVariant;
108
- readonly harness: OrchestratorV2ProviderReplayHarness<Transcript, Error>;
109
- readonly transformTranscript?: (transcript: ProviderReplayTranscript) => ProviderReplayTranscript;
110
-}) {
111
- const recordedTranscript = yield* readTranscript(input.driver.transcriptFile);
112
- const rawTranscript = input.transformTranscript?.(recordedTranscript) ?? recordedTranscript;
113
- const replayTranscript = materializeReplayTranscriptRuntimeInstructions(
114
- transcriptEntriesThroughLabel(rawTranscript, input.driver.transcriptEntriesThroughLabel),
115
- { driver: input.driver.driver, model: input.driver.modelSelection.model },
116
- );
117
- const fixtureInput = input.buildInput();
118
- const workspace = yield* checkpointWorkspace(input.fixtureName, fixtureInput.workspaceFiles);
119
- const transcript = yield* input.harness.decodeTranscript(
120
- input.driver.driver === "codex"
121
- ? materializeReplayTranscriptWorkspace(replayTranscript, workspace)
122
- : input.driver.driver === "muse"
123
- ? yield* materializeMuseReplayWorkspace(replayTranscript, workspace)
124
- : replayTranscript,
125
- );
126
- const materialized = yield* materializeFixtureInput({
127
- scenario: input.fixtureName,
128
- fixtureInput,
129
- driver: input.driver.driver,
130
- modelSelection: input.driver.modelSelection,
131
- }).pipe(Effect.provide(IdAllocator.layer), provideDeterministicTestRuntime);
132
- const scenario = {
133
- name: `${input.fixtureName}/${input.driver.driver}`,
134
- transcript,
135
- commands: materialized.commands,
136
- steps: materialized.steps,
137
- projectionThreadIds: materialized.projectionThreadIds,
138
- runtimePolicyOverride: {
139
- ...input.driver.runtimePolicyOverride,
140
- cwd: workspace,
141
- },
142
- };
143
-
144
- const result = yield* runOrchestratorV2ProviderReplayScenario(
145
- scenario,
146
- input.harness,
147
- input.driver.runContinuationWorker === true ? { runContinuationWorker: true } : {},
148
- ).pipe(provideDeterministicTestRuntime);
149
- input.driver.assertOutput(result, transcript);
150
- assertProviderNativeSubagentRootTurns(result);
151
- const expectedAbsentWorkspacePaths = input.driver.expectedAbsentWorkspacePaths;
152
- if (expectedAbsentWorkspacePaths !== undefined) {
153
- yield* Effect.gen(function* () {
154
- const fs = yield* FileSystem.FileSystem;
155
- const path = yield* Path.Path;
156
- for (const relativePath of expectedAbsentWorkspacePaths) {
157
- assert.isFalse(
158
- yield* fs.exists(path.join(workspace, relativePath)),
159
- `${input.fixtureName}/${input.driver.driver} must not create ${relativePath} in the replay workspace`,
160
- );
161
- }
162
- }).pipe(Effect.provide(NodeServices.layer));
163
- }
164
- assert.isFalse(
165
- result.domainEvents.some(isStreamingAssistantEvent),
166
- "buffered delivery must not persist streaming assistant artifacts",
167
- );
168
- const projectionThreadId = materialized.projectionThreadIds[0];
169
- assert.isDefined(projectionThreadId);
170
- const projection = result.projections.get(projectionThreadId);
171
- assert.isDefined(projection);
172
- const latestRun = projection.runs.at(-1);
173
- assert.deepEqual(latestRun?.modelSelection, input.driver.modelSelection);
174
- if (projection.runs.some((run) => run.status === "completed")) {
175
- const threadStartCheckpoint = projection.checkpoints.find(
176
- (checkpoint) => checkpoint.ordinalWithinScope === 0 && checkpoint.appRunOrdinal === null,
177
- );
178
- assert.isDefined(
179
- threadStartCheckpoint,
180
- "completed threads must retain an addressable thread-start checkpoint",
181
- );
182
- assert.equal(threadStartCheckpoint.status, "ready");
183
- }
184
- return result;
185
-});
186
-
187
-function runFixtureProviderWithRegisteredHarness(input: {
188
- readonly fixtureName: string;
189
- readonly buildInput: () => OrchestratorFixtureInput;
190
- readonly driver: ProviderOrchestratorReplayVariant;
191
- readonly transformTranscript?: (transcript: ProviderReplayTranscript) => ProviderReplayTranscript;
192
-}) {
193
- switch (input.driver.driver) {
194
- case "codex":
195
- return runFixtureProvider({
196
- ...input,
197
- harness: CodexOrchestratorReplayHarness,
198
- }).pipe(Effect.mapError(normalizeTestError), Effect.scoped);
199
- case "claudeAgent":
200
- return runFixtureProvider({
201
- ...input,
202
- harness: ClaudeOrchestratorReplayHarness,
203
- }).pipe(Effect.mapError(normalizeTestError), Effect.scoped);
204
- case "cursor":
205
- return runFixtureProvider({
206
- ...input,
207
- harness: CursorOrchestratorReplayHarness,
208
- }).pipe(Effect.mapError(normalizeTestError), Effect.scoped);
209
- case "grok":
210
- return runFixtureProvider({
211
- ...input,
212
- harness: GrokOrchestratorReplayHarness,
213
- }).pipe(Effect.mapError(normalizeTestError), Effect.scoped);
214
- case "acpRegistry":
215
- return runFixtureProvider({
216
- ...input,
217
- harness: AcpRegistryOrchestratorReplayHarness,
218
- }).pipe(Effect.mapError(normalizeTestError), Effect.scoped);
219
- case "opencode":
220
- return readTranscript(input.driver.transcriptFile).pipe(
221
- Effect.flatMap((transcript) =>
222
- transcript.protocol === OPENCODE2_HTTP_PROTOCOL
223
- ? runFixtureProvider({ ...input, harness: OpenCode2OrchestratorReplayHarness })
224
- : runFixtureProvider({ ...input, harness: OpenCodeOrchestratorReplayHarness }),
225
- ),
226
- Effect.mapError(normalizeTestError),
227
- Effect.scoped,
228
- );
229
- case "pi":
230
- return runFixtureProvider({
231
- ...input,
232
- harness: PiOrchestratorReplayHarness,
233
- }).pipe(Effect.mapError(normalizeTestError), Effect.scoped);
234
- case "muse":
235
- return runFixtureProvider({
236
- ...input,
237
- harness: MuseOrchestratorReplayHarness,
238
- }).pipe(Effect.mapError(normalizeTestError), Effect.scoped);
239
- default:
240
- return Effect.die(
241
- new Error(`No replay harness registered for provider ${input.driver.driver}.`),
242
- );
243
- }
244
-}
245
-
246
-describe("orchestrator replay fixtures", () => {
247
- it.effect.each([
248
- { name: "open", firstTurn: false, code: -32602 },
249
- { name: "switch", firstTurn: true, code: -32602 },
250
- { name: "authentication", firstTurn: false, code: -32000 },
251
- ])("preserves the ACP model-write $name failure through orchestration", (testCase) =>
252
- Effect.gen(function* () {
253
- const name = `acp_model_write_${testCase.name}`;
254
- const sessionId = "grok-replay-session-1";
255
- const rejectedModel = "replay-model-b";
256
- const modelSelection = {
257
- instanceId: ProviderInstanceId.make("acpRegistry"),
258
- model: rejectedModel,
259
- };
260
- const recorded = yield* readTranscript(
261
- new URL("./fixtures/simple/registry_transcript.ndjson", import.meta.url),
262
- );
263
- const sessionIndex = recorded.entries.findIndex(
264
- (entry) => "label" in entry && entry.label === "session.created",
265
- );
266
- assert.isAtLeast(sessionIndex, 0);
267
- const transcript = {
268
- ...recorded,
269
- scenario: name,
270
- entries: [
271
- ...recorded.entries.slice(0, sessionIndex),
272
- {
273
- type: "emit_inbound",
274
- label: "session.created",
275
- frame: {
276
- kind: "response",
277
- method: "session/new",
278
- result: {
279
- sessionId,
280
- configOptions: [
281
- {
282
- id: "model",
283
- name: "Model",
284
- category: "model",
285
- type: "select",
286
- currentValue: "replay-model-a",
287
- options: [
288
- { value: "replay-model-a", name: "Model A" },
289
- { value: rejectedModel, name: "Model B" },
290
- ],
291
- },
292
- ],
293
- },
294
- },
295
- },
296
- ...(testCase.firstTurn
297
- ? recorded.entries
298
- .slice(sessionIndex + 1)
299
- .filter((entry) => entry.type !== "runtime_exit")
300
- : []),
301
- {
302
- type: "expect_outbound",
303
- label: "session.set_model",
304
- frame: {
305
- kind: "request",
306
- method: "session/set_config_option",
307
- params: { sessionId, configId: "model", value: rejectedModel },
308
- },
309
- },
310
- {
311
- type: "emit_inbound",
312
- label: "session.set_model.rejected",
313
- frame: {
314
- kind: "response",
315
- method: "session/set_config_option",
316
- error: {
317
- code: testCase.code,
318
- message: testCase.code === -32602 ? "Invalid provider" : "Authentication required",
319
- },
320
- },
321
- },
322
- ],
323
- } satisfies ProviderReplayTranscript;
324
- const materialized = yield* materializeFixtureInput({
325
- scenario: name,
326
- fixtureInput: {
327
- steps: Array.from({ length: testCase.firstTurn ? 2 : 1 }, () => ({
328
- type: "message" as const,
329
- text: SIMPLE_PROMPT,
330
- })),
331
- },
332
- driver: ProviderDriverKind.make("acpRegistry"),
333
- modelSelection: {
334
- ...modelSelection,
335
- model: testCase.firstTurn ? "replay-model-a" : rejectedModel,
336
- },
337
- });
338
- const scenario = {
339
- ...materialized,
340
- name,
341
- transcript: yield* AcpRegistryOrchestratorReplayHarness.decodeTranscript(
342
- materializeReplayTranscriptRuntimeInstructions(transcript, {
343
- driver: ProviderDriverKind.make("acpRegistry"),
344
- model: "replay-model-a",
345
- }),
346
- ),
347
- runtimePolicyOverride: { cwd: yield* checkpointWorkspace(name) },
348
- };
349
- yield* Effect.gen(function* () {
350
- const orchestrator = yield* Orchestrator.OrchestratorV2;
351
- const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
352
- if (testCase.firstTurn) yield* EffectWorker.runDaemon.pipe(Effect.forkScoped);
353
- for (const [index, command] of materialized.commands.entries()) {
354
- const isFirstTurn =
355
- testCase.firstTurn && command.type === "message.dispatch" && index === 1;
356
- yield* orchestrator.dispatch(
357
- command.type === "message.dispatch" && !isFirstTurn
358
- ? { ...command, modelSelection }
359
- : command,
360
- );
361
- if (isFirstTurn) {
362
- // Wait for the committed terminal event before switching models.
363
- yield* orchestrator.streamStoredEvents.pipe(
364
- Stream.filter(
365
- ({ event }) => event.type === "run.updated" && event.payload.status === "completed",
366
- ),
367
- Stream.runHead,
368
- );
369
- }
370
- }
371
- if (testCase.firstTurn) {
372
- yield* orchestrator.streamStoredEvents.pipe(
373
- Stream.filter(
374
- ({ event }) => event.type === "run.updated" && event.payload.status === "failed",
375
- ),
376
- Stream.runHead,
377
- );
378
- } else {
379
- // Failed opens retry on the outbox schedule. Advance to each recorded
380
- // deadline, draining effects instead of polling the projection.
381
- for (let attempt = 0; attempt < 10; attempt++) {
382
- yield* worker.drain();
383
- const next = yield* worker.nextClaimableAt;
384
- if (Option.isNone(next)) break;
385
- const now = yield* DateTime.now;
386
- yield* TestClock.adjust(
387
- Math.max(0, DateTime.toEpochMillis(next.value) - DateTime.toEpochMillis(now)),
388
- );
389
- }
390
- }
391
- const projection = yield* orchestrator.getThreadProjection(
392
- materialized.projectionThreadIds[0]!,
393
- );
394
- assert.deepEqual(
395
- projection.runs.map((run) => run.status),
396
- testCase.firstTurn ? ["completed", "failed"] : ["failed"],
397
- );
398
- const errors = projection.turnItems.filter((item) => item.type === "error");
399
- assert.lengthOf(errors, 1);
400
- assert.equal(errors[0]?.failure.class, "provider_error");
401
- if (testCase.code === -32602) {
402
- assert.equal(
403
- errors[0]?.failure.message,
404
- 'The agent did not accept the model "replay-model-b". Choose another model for this provider, then retry.',
405
- );
406
- } else {
407
- assert.match(
408
- errors[0]?.failure.message ?? "",
409
- /^Failed to open acpRegistry provider session \S+\.$/u,
410
- );
411
- }
412
- }).pipe(
413
- Effect.provide(
414
- layerProviderReplay(scenario, AcpRegistryOrchestratorReplayHarness, {
415
- runEffectWorker: false,
416
- }),
417
- ),
418
- );
419
- }).pipe(
420
- Effect.scoped,
421
- provideDeterministicTestRuntime,
422
- Effect.provide(Layer.merge(IdAllocator.layer, NodeServices.layer)),
423
- ),
424
- );
425
-
426
- it.effect.each(
427
- ORCHESTRATOR_REPLAY_FIXTURES.flatMap((fixture) =>
428
- fixture.providers.map(
429
- (provider) => [fixture.name, provider.driver, fixture, provider] as const,
430
- ),
431
- ),
432
- )("runs %s/%s through OrchestratorV2 using deterministic replay", ([, , fixture, provider]) =>
433
- runFixtureProviderWithRegisteredHarness({
434
- fixtureName: fixture.name,
435
- buildInput: fixture.buildInput,
436
- driver: provider,
437
- }),
438
- );
439
-
440
- const steeringFixture = ORCHESTRATOR_REPLAY_FIXTURES.find(
441
- (fixture) => fixture.name === "message_steering",
442
- );
443
- const cursorSteeringProvider = steeringFixture?.providers.find(
444
- (provider) => provider.driver === "cursor",
445
- );
446
- if (cursorSteeringProvider !== undefined) {
447
- it.effect("executes explicit Cursor restart_active through the recorded SDK boundary", () =>
448
- runFixtureProviderWithRegisteredHarness({
449
- fixtureName: "message_steering",
450
- buildInput: messageRestartInput,
451
- driver: cursorSteeringProvider,
452
- }),
453
- );
454
- }
455
-
456
- it.effect.each(
457
- (
458
- [
459
- ["opencode2_subagent", "session.execution.started.2"],
460
- ["opencode2_background", "session.execution.started.3"],
461
- ] as const
462
- ).flatMap(([fixtureName, label]) => {
463
- const fixture = ORCHESTRATOR_REPLAY_FIXTURES.find(
464
- (candidate) => candidate.name === fixtureName,
465
- );
466
- const provider = fixture?.providers[0];
467
- return fixture === undefined || provider === undefined
468
- ? []
469
- : [[fixtureName, label, fixture, provider] as const];
470
- }),
471
- )(
472
- "runs %s when %s is an execution start this build cannot decode",
473
- ([fixtureName, label, fixture, provider]) =>
474
- runFixtureProviderWithRegisteredHarness({
475
- fixtureName,
476
- buildInput: fixture.buildInput,
477
- driver: provider,
478
- transformTranscript: (transcript) => ({
479
- ...transcript,
480
- entries: transcript.entries.map((entry) =>
481
- entry.type === "emit_inbound" && entry.label === label
482
- ? undecodableEvent(entry)
483
- : entry,
484
- ),
485
- }),
486
- }),
487
- );
488
-
489
- const afterRootFixture = ORCHESTRATOR_REPLAY_FIXTURES.find(
490
- (candidate) => candidate.name === "claude_background_subagent_after_root",
491
- );
492
- const afterRootProvider = afterRootFixture?.providers[0];
493
- if (afterRootFixture !== undefined && afterRootProvider !== undefined) {
494
- const OPEN_TOOL_ID = "toolu_01QDa5jV5g1H9h6QyDfeJohD";
495
- const SUBAGENT_TASK_ID = "a2995bfced8019363";
496
-
497
- const moveOpenToolBeforeRootResult = (
498
- move: boolean,
499
- entries: ReadonlyArray<ProviderReplayEntry>,
500
- ): ReadonlyArray<ProviderReplayEntry> => {
501
- if (!move) return entries;
502
- const toolIndex = entries.findIndex(
503
- (entry) =>
504
- entry.type === "emit_inbound" &&
505
- (entry.frame as Record<string, any>).message?.content?.[0]?.id === OPEN_TOOL_ID,
506
- );
507
- const resultIndex = entries.findIndex(
508
- (entry) => entry.type === "emit_inbound" && entry.label === "result",
509
- );
510
- if (toolIndex < 0 || resultIndex < 0) throw new Error("transcript shape changed");
511
- const toolEntry = entries[toolIndex]!;
512
- const rest = entries.filter((_, index) => index !== toolIndex);
513
- return [...rest.slice(0, resultIndex), toolEntry, ...rest.slice(resultIndex)];
514
- };
515
- // A subagent's foreground Bash run during the root's turn is tailed like a
516
- // root call's, so the child's command row streams live.
517
- const SUBAGENT_BASH_ID = "toolu_01JMN3SSQCHzUTNua9AWdkiH";
518
- it.effect("streams a subagent's foreground Bash output live during the root turn", () =>
519
- runFixtureProviderWithRegisteredHarness({
520
- fixtureName: afterRootFixture.name,
521
- buildInput: afterRootFixture.buildInput,
522
- driver: {
523
- ...afterRootProvider,
524
- assertOutput: (result) => {
525
- const childThreadId = projectionFor(result, afterRootFixture.name).subagents[0]
526
- ?.childThreadId;
527
- assert.exists(childThreadId);
528
- const command = result.projections
529
- .get(childThreadId)
530
- ?.turnItems.find(
531
- (item) =>
532
- item.type === "command_execution" &&
533
- JSON.stringify(item.input).includes("SUB_DONE_1"),
534
- );
535
- assert.exists(command);
536
- const streamed = result.commandOutput?.get(command.id);
537
- assert.exists(streamed, "the subagent's command must stream its output");
538
- assert.include(streamed.output.text, "SUB_DONE_1");
539
- },
540
- },
541
- transformTranscript: (transcript) => {
542
- const isBashFrame = (entry: ProviderReplayEntry) => {
543
- if (entry.type !== "emit_inbound") return false;
544
- const frame = entry.frame as Record<string, any>;
545
- return (
546
- frame.message?.content?.[0]?.id === SUBAGENT_BASH_ID ||
547
- frame.message?.content?.[0]?.tool_use_id === SUBAGENT_BASH_ID ||
548
- (frame.type === "system" && frame.tool_use_id === SUBAGENT_BASH_ID)
549
- );
550
- };
551
- // Recorded without the structured result the replay writes output files from.
552
- const bashFrames = transcript.entries.filter(isBashFrame).map((entry) => {
553
- const frame = (entry as { readonly frame: Record<string, any> }).frame;
554
- return frame.type === "user"
555
- ? {
556
- ...entry,
557
- frame: { ...frame, tool_use_result: { stdout: "SUB_DONE_1\n", stderr: "" } },
558
- }
559
- : entry;
560
- });
561
- const rest = transcript.entries.filter((entry) => !isBashFrame(entry));
562
- const resultIndex = rest.findIndex(
563
- (entry) => entry.type === "emit_inbound" && entry.label === "result",
564
- );
565
- assert.lengthOf(bashFrames, 4, "transcript shape changed");
566
- assert.isAtLeast(resultIndex, 0, "transcript shape changed");
567
- return {
568
- ...transcript,
569
- entries: [...rest.slice(0, resultIndex), ...bashFrames, ...rest.slice(resultIndex)],
570
- };
571
- },
572
- }),
573
- );
574
-
575
- // regression: #16486 a background subagent's tool left open while its parent is idle never settled
576
- it.effect.each([
577
- ["WebFetch", "while root is idle", "stopped"],
578
- ["Write", "before root settles", "failed"],
579
- ] as const)(
580
- "ends a background subagent's open %s (opened %s) when its notification is %s",
581
- ([tool, opens, notificationStatus]) =>
582
- runFixtureProviderWithRegisteredHarness({
583
- fixtureName: afterRootFixture.name,
584
- buildInput: afterRootFixture.buildInput,
585
- driver: {
586
- ...afterRootProvider,
587
- assertOutput: (result) => {
588
- const parent = projectionFor(result, afterRootFixture.name);
589
- assert.deepEqual(
590
- parent.runs.map((run) => run.status),
591
- ["completed", "completed"],
592
- );
593
- const subagent = parent.subagents[0];
594
- assert.equal(
595
- subagent?.status,
596
- notificationStatus === "stopped" ? "cancelled" : "failed",
597
- );
598
- const childThreadId = subagent?.childThreadId;
599
- assert.exists(childThreadId);
600
- const child = result.projections.get(childThreadId);
601
- assert.exists(child);
602
- const fetches = child.turnItems.filter(
603
- (item) => item.type === (tool === "WebFetch" ? "web_search" : "file_change"),
604
- );
605
- assert.lengthOf(fetches, 1, `the ${tool} reached the child thread`);
606
- assert.deepEqual(
607
- {
608
- openTool: fetches.map((item) => item.status),
609
- openChildNodes: child.nodes
610
- .filter(
611
- (node) =>
612
- node.status === "running" ||
613
- node.status === "pending" ||
614
- node.status === "waiting",
615
- )
616
- .map((node) => `${node.kind}:${node.status}`),
617
- },
618
- {
619
- openTool: [notificationStatus === "stopped" ? "interrupted" : "failed"],
620
- openChildNodes: [],
621
- },
622
- );
623
- },
624
- },
625
- transformTranscript: (transcript) => ({
626
- ...transcript,
627
- entries: moveOpenToolBeforeRootResult(
628
- opens === "before root settles",
629
- transcript.entries.flatMap((entry): ReadonlyArray<ProviderReplayEntry> => {
630
- if (entry.type !== "emit_inbound") return [entry];
631
- const frame = entry.frame as Record<string, any>;
632
- if (
633
- frame.type === "assistant" &&
634
- frame.message?.content?.[0]?.id === OPEN_TOOL_ID
635
- ) {
636
- return [
637
- {
638
- ...entry,
639
- frame: {
640
- ...frame,
641
- message: {
642
- ...frame.message,
643
- content: [
644
- {
645
- ...frame.message.content[0],
646
- name: tool,
647
- input:
648
- tool === "WebFetch"
649
- ? { url: "https://example.com", prompt: "Summarize" }
650
- : { file_path: "notes.txt", content: "SUB_DONE_2" },
651
- },
652
- ],
653
- },
654
- },
655
- },
656
- ];
657
- }
658
- if (
659
- (frame.type === "system" &&
660
- (frame.subtype === "task_started" || frame.subtype === "task_notification") &&
661
- frame.tool_use_id === OPEN_TOOL_ID) ||
662
- (frame.type === "user" &&
663
- frame.message?.content?.[0]?.tool_use_id === OPEN_TOOL_ID) ||
664
- (frame.type === "assistant" &&
665
- frame.parent_tool_use_id !== null &&
666
- frame.message?.content?.[0]?.text === "SUB_FINAL_REPORT")
667
- ) {
668
- return [];
669
- }
670
- if (frame.type === "system" && frame.subtype === "task_updated") {
671
- return [
672
- {
673
- ...entry,
674
- frame: {
675
- ...frame,
676
- patch: {
677
- ...frame.patch,
678
- status: notificationStatus === "stopped" ? "killed" : "failed",
679
- },
680
- },
681
- },
682
- ];
683
- }
684
- if (
685
- frame.type === "system" &&
686
- frame.subtype === "task_notification" &&
687
- frame.task_id === SUBAGENT_TASK_ID
688
- ) {
689
- return [{ ...entry, frame: { ...frame, status: notificationStatus } }];
690
- }
691
- return [entry];
692
- }),
693
- ),
694
- }),
695
- }),
696
- );
697
- }
698
-});
699
-
700
-function undecodableEvent(entry: ProviderReplayEntry): ProviderReplayEntry {
701
- if (entry.type !== "emit_inbound") return entry;
702
- const frame = entry.frame as { readonly event: Record<string, unknown> };
703
- return { ...entry, frame: { ...frame, event: { ...frame.event, durable: "not-an-envelope" } } };
704
-}
apps/server/src/orchestration-v2/testkit/PiReplayFixtures.integration.test.ts
new
+10
@@ -0,0 +1,10 @@
1
+import { describe } from "@effect/vitest";
2
+
3
+import { PiOrchestratorReplayHarness } from "../Adapters/PiAdapterV2.testkit.ts";
4
+import { itRunsReplayFixtures, runReplayFixture } from "./ProviderReplayFixtures.ts";
5
+
6
+describe("Pi replay fixtures", () => {
7
+ itRunsReplayFixtures("pi", (input) =>
8
+ runReplayFixture({ ...input, harness: PiOrchestratorReplayHarness }),
9
+ );
10
+});
apps/server/src/orchestration-v2/testkit/ProviderReplayFixtures.ts
new
+219
@@ -0,0 +1,219 @@
1
+import * as NodeServices from "@effect/platform-node/NodeServices";
2
+import { assert, it } from "@effect/vitest";
3
+import type { OrchestrationV2DomainEvent, ProviderReplayTranscript } from "@t3tools/contracts";
4
+import * as Effect from "effect/Effect";
5
+import * as FileSystem from "effect/FileSystem";
6
+import * as Path from "effect/Path";
7
+
8
+import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
9
+import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace";
10
+import { readProviderReplayTranscript } from "@t3tools/provider-testing/replayTranscript";
11
+import { provideDeterministicTestRuntime } from "./DeterministicRuntime.ts";
12
+import { ORCHESTRATOR_REPLAY_FIXTURES } from "./fixtures/index.ts";
13
+import {
14
+ assertProviderNativeSubagentRootTurns,
15
+ materializeFixtureInput,
16
+ type OrchestratorFixtureInput,
17
+ type ProviderOrchestratorReplayVariant,
18
+} from "./fixtures/shared.ts";
19
+import {
20
+ runOrchestratorV2ProviderReplayScenario,
21
+ type OrchestratorV2ProviderReplayHarness,
22
+} from "./ProviderReplayHarness.ts";
23
+import { materializeReplayTranscriptRuntimeInstructions } from "./ReplayRuntimeInstructions.ts";
24
+
25
+// Each driver's fixtures run from its own <Provider>ReplayFixtures.integration.test.ts,
26
+// so a focused run loads one adapter. A fixture for a driver missing here would
27
+// silently never run, so loading this module fails instead.
28
+const DRIVERS_WITH_REPLAY_FIXTURE_FILES = new Set([
29
+ "acpRegistry",
30
+ "claudeAgent",
31
+ "codex",
32
+ "cursor",
33
+ "grok",
34
+ "muse",
35
+ "opencode",
36
+ "pi",
37
+]);
38
+for (const fixture of ORCHESTRATOR_REPLAY_FIXTURES) {
39
+ for (const provider of fixture.providers) {
40
+ if (!DRIVERS_WITH_REPLAY_FIXTURE_FILES.has(provider.driver)) {
41
+ throw new Error(
42
+ `${fixture.name}/${provider.driver} has no ReplayFixtures test file to run it.`,
43
+ );
44
+ }
45
+ }
46
+}
47
+
48
+export const readReplayFixtureTranscript = Effect.fn("readOrchestratorReplayTranscript")(function* (
49
+ file: URL,
50
+) {
51
+ return yield* readProviderReplayTranscript(file);
52
+}, Effect.provide(NodeServices.layer));
53
+
54
+function normalizeTestError(cause: unknown): Error {
55
+ return cause instanceof Error ? cause : new Error(String(cause));
56
+}
57
+
58
+function transcriptEntriesThroughLabel(
59
+ transcript: ProviderReplayTranscript,
60
+ label: string | undefined,
61
+): ProviderReplayTranscript {
62
+ if (label === undefined) {
63
+ return transcript;
64
+ }
65
+ const entryIndex = transcript.entries.findIndex(
66
+ (entry) => entry.type === "emit_inbound" && entry.label === label,
67
+ );
68
+ if (entryIndex === -1) {
69
+ throw new Error(`${transcript.scenario} is missing inbound transcript label ${label}.`);
70
+ }
71
+ return {
72
+ ...transcript,
73
+ entries: transcript.entries.slice(0, entryIndex + 1),
74
+ };
75
+}
76
+
77
+function isStreamingAssistantEvent(event: OrchestrationV2DomainEvent): boolean {
78
+ switch (event.type) {
79
+ case "node.updated":
80
+ return event.payload.kind === "assistant_message" && event.payload.status === "running";
81
+ case "message.updated":
82
+ return event.payload.role === "assistant" && event.payload.streaming;
83
+ case "turn-item.updated":
84
+ return event.payload.type === "assistant_message" && event.payload.streaming;
85
+ default:
86
+ return false;
87
+ }
88
+}
89
+
90
+export interface ReplayFixtureCase {
91
+ readonly fixtureName: string;
92
+ readonly buildInput: () => OrchestratorFixtureInput;
93
+ readonly driver: ProviderOrchestratorReplayVariant;
94
+ readonly transformTranscript?: (transcript: ProviderReplayTranscript) => ProviderReplayTranscript;
95
+}
96
+
97
+export interface ReplayFixtureRun<
98
+ Transcript extends ProviderReplayTranscript,
99
+ Error,
100
+ WorkspaceError = never,
101
+> extends ReplayFixtureCase {
102
+ readonly harness: OrchestratorV2ProviderReplayHarness<Transcript, Error>;
103
+ /** Rewrites recorded workspace paths to the replay workspace, for drivers that record them. */
104
+ readonly materializeWorkspace?: (
105
+ transcript: ProviderReplayTranscript,
106
+ workspace: string,
107
+ ) => Effect.Effect<ProviderReplayTranscript, WorkspaceError>;
108
+}
109
+
110
+const runFixtureProvider = Effect.fn("runOrchestratorReplayFixture")(function* <
111
+ Transcript extends ProviderReplayTranscript,
112
+ Error,
113
+ WorkspaceError,
114
+>(input: ReplayFixtureRun<Transcript, Error, WorkspaceError>) {
115
+ const recordedTranscript = yield* readReplayFixtureTranscript(input.driver.transcriptFile);
116
+ const rawTranscript = input.transformTranscript?.(recordedTranscript) ?? recordedTranscript;
117
+ const replayTranscript = materializeReplayTranscriptRuntimeInstructions(
118
+ transcriptEntriesThroughLabel(rawTranscript, input.driver.transcriptEntriesThroughLabel),
119
+ { driver: input.driver.driver, model: input.driver.modelSelection.model },
120
+ );
121
+ const fixtureInput = input.buildInput();
122
+ const workspace = yield* checkpointWorkspace(input.fixtureName, fixtureInput.workspaceFiles);
123
+ const transcript = yield* input.harness.decodeTranscript(
124
+ input.materializeWorkspace === undefined
125
+ ? replayTranscript
126
+ : yield* input.materializeWorkspace(replayTranscript, workspace),
127
+ );
128
+ const materialized = yield* materializeFixtureInput({
129
+ scenario: input.fixtureName,
130
+ fixtureInput,
131
+ driver: input.driver.driver,
132
+ modelSelection: input.driver.modelSelection,
133
+ }).pipe(Effect.provide(IdAllocator.layer), provideDeterministicTestRuntime);
134
+ const scenario = {
135
+ name: `${input.fixtureName}/${input.driver.driver}`,
136
+ transcript,
137
+ commands: materialized.commands,
138
+ steps: materialized.steps,
139
+ projectionThreadIds: materialized.projectionThreadIds,
140
+ runtimePolicyOverride: {
141
+ ...input.driver.runtimePolicyOverride,
142
+ cwd: workspace,
143
+ },
144
+ };
145
+
146
+ const result = yield* runOrchestratorV2ProviderReplayScenario(
147
+ scenario,
148
+ input.harness,
149
+ input.driver.runContinuationWorker === true ? { runContinuationWorker: true } : {},
150
+ ).pipe(provideDeterministicTestRuntime);
151
+ input.driver.assertOutput(result, transcript);
152
+ assertProviderNativeSubagentRootTurns(result);
153
+ const expectedAbsentWorkspacePaths = input.driver.expectedAbsentWorkspacePaths;
154
+ if (expectedAbsentWorkspacePaths !== undefined) {
155
+ yield* Effect.gen(function* () {
156
+ const fs = yield* FileSystem.FileSystem;
157
+ const path = yield* Path.Path;
158
+ for (const relativePath of expectedAbsentWorkspacePaths) {
159
+ assert.isFalse(
160
+ yield* fs.exists(path.join(workspace, relativePath)),
161
+ `${input.fixtureName}/${input.driver.driver} must not create ${relativePath} in the replay workspace`,
162
+ );
163
+ }
164
+ }).pipe(Effect.provide(NodeServices.layer));
165
+ }
166
+ assert.isFalse(
167
+ result.domainEvents.some(isStreamingAssistantEvent),
168
+ "buffered delivery must not persist streaming assistant artifacts",
169
+ );
170
+ const projectionThreadId = materialized.projectionThreadIds[0];
171
+ assert.isDefined(projectionThreadId);
172
+ const projection = result.projections.get(projectionThreadId);
173
+ assert.isDefined(projection);
174
+ const latestRun = projection.runs.at(-1);
175
+ assert.deepEqual(latestRun?.modelSelection, input.driver.modelSelection);
176
+ if (projection.runs.some((run) => run.status === "completed")) {
177
+ const threadStartCheckpoint = projection.checkpoints.find(
178
+ (checkpoint) => checkpoint.ordinalWithinScope === 0 && checkpoint.appRunOrdinal === null,
179
+ );
180
+ assert.isDefined(
181
+ threadStartCheckpoint,
182
+ "completed threads must retain an addressable thread-start checkpoint",
183
+ );
184
+ assert.equal(threadStartCheckpoint.status, "ready");
185
+ }
186
+ return result;
187
+});
188
+
189
+/** Replays one fixture variant through OrchestratorV2 and runs the shared output checks. */
190
+export function runReplayFixture<
191
+ Transcript extends ProviderReplayTranscript,
192
+ Error,
193
+ WorkspaceError = never,
194
+>(input: ReplayFixtureRun<Transcript, Error, WorkspaceError>) {
195
+ return runFixtureProvider(input).pipe(Effect.mapError(normalizeTestError), Effect.scoped);
196
+}
197
+
198
+export function findReplayFixture(name: string) {
199
+ return ORCHESTRATOR_REPLAY_FIXTURES.find((fixture) => fixture.name === name);
200
+}
201
+
202
+/**
203
+ * Registers one test per recorded fixture variant of `driver`. Call it inside the
204
+ * provider file's `describe`. `run` picks the harness for each variant.
205
+ */
206
+export function itRunsReplayFixtures<E>(
207
+ driver: string,
208
+ run: (input: ReplayFixtureCase) => Effect.Effect<unknown, E>,
209
+) {
210
+ it.effect.each(
211
+ ORCHESTRATOR_REPLAY_FIXTURES.flatMap((fixture) =>
212
+ fixture.providers
213
+ .filter((provider) => provider.driver === driver)
214
+ .map((provider) => [fixture.name, provider.driver, fixture, provider] as const),
215
+ ),
216
+ )("runs %s/%s through OrchestratorV2 using deterministic replay", ([, , fixture, provider]) =>
217
+ run({ fixtureName: fixture.name, buildInput: fixture.buildInput, driver: provider }),
218
+ );
219
+}
apps/server/src/testUtils/shardWeights.json
+8
-2
@@ -123,10 +123,16 @@
123
"src/orchestration-v2/legacy/LegacyV1Cutover.integration.test.ts": 1.4,
124
"src/orchestration-v2/legacy/LegacyV1ThreadImporter.test.ts": 0.5,
125
"src/orchestration-v2/runtimeLayer.test.ts": 2.8,
126
- "src/orchestration-v2/testkit/ClaudeReplayFixtures.integration.test.ts": 2.7,
127
- "src/orchestration-v2/testkit/OrchestratorReplayFixtures.integration.test.ts": 44.6,
126
+ "src/orchestration-v2/testkit/AcpRegistryReplayFixtures.integration.test.ts": 4.8,
127
+ "src/orchestration-v2/testkit/ClaudeReplayFixtures.integration.test.ts": 13.7,
128
+ "src/orchestration-v2/testkit/CodexReplayFixtures.integration.test.ts": 6.4,
129
+ "src/orchestration-v2/testkit/CursorReplayFixtures.integration.test.ts": 4.8,
130
+ "src/orchestration-v2/testkit/GrokReplayFixtures.integration.test.ts": 5.1,
131
+ "src/orchestration-v2/testkit/MuseReplayFixtures.integration.test.ts": 2.4,
132
+ "src/orchestration-v2/testkit/OpenCodeReplayFixtures.integration.test.ts": 5.8,
133
"src/orchestration-v2/testkit/OrchestratorReplayRecovery.integration.test.ts": 2.2,
134
"src/orchestration-v2/testkit/OrchestratorReplayRestartBackgroundNote.integration.test.ts": 2.1,
135
+ "src/orchestration-v2/testkit/PiReplayFixtures.integration.test.ts": 4.3,
136
"src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts": 16.1,
137
"src/orchestration-v2/testkit/ThreadFork.integration.test.ts": 2.9,
138
"src/orchestration-v2/testkit/ThreadMergeBack.integration.test.ts": 5.1,