The best way to control Claude Code, Codex and more from any computer or phone

main
ts 232 lines 9.11 KB
Raw
1 import * as NodeServices from "@effect/platform-node/NodeServices";
2 import { assert, it } from "@effect/vitest";
3 import type { OrchestrationV2DomainEvent, ProviderReplayTranscript } from "@t3tools/contracts";
4 import * as Effect from "effect/Effect";
5 import * as FileSystem from "effect/FileSystem";
6 import * as Path from "effect/Path";
7
8 import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
9 import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace";
10 import { readProviderReplayTranscript } from "@t3tools/provider-testing/replayTranscript";
11 import type { ProviderInstanceRegistry } from "../../provider/ProviderInstanceRegistry.ts";
12 import { provideDeterministicTestRuntime } from "./DeterministicRuntime.ts";
13 import { ORCHESTRATOR_REPLAY_FIXTURES } from "./fixtures/index.ts";
14 import {
15 assertProviderNativeSubagentRootTurns,
16 materializeFixtureInput,
17 type OrchestratorFixtureInput,
18 type ProviderOrchestratorReplayVariant,
19 } from "./fixtures/shared.ts";
20 import {
21 runOrchestratorV2ProviderReplayScenario,
22 type OrchestratorV2ProviderReplayHarness,
23 } from "./ProviderReplayHarness.ts";
24 import { materializeReplayTranscriptRuntimeInstructions } from "./ReplayRuntimeInstructions.ts";
25
26 // Each driver's fixtures run from its own <Provider>ReplayFixtures.integration.test.ts,
27 // so a focused run loads one adapter. A fixture for a driver missing here would
28 // silently never run, so loading this module fails instead.
29 const DRIVERS_WITH_REPLAY_FIXTURE_FILES = new Set([
30 "acpRegistry",
31 "claudeAgent",
32 "codex",
33 "cursor",
34 "fx",
35 "grok",
36 "kiro",
37 "muse",
38 "opencode",
39 "pi",
40 ]);
41 for (const fixture of ORCHESTRATOR_REPLAY_FIXTURES) {
42 for (const provider of fixture.providers) {
43 if (!DRIVERS_WITH_REPLAY_FIXTURE_FILES.has(provider.driver)) {
44 throw new Error(
45 `${fixture.name}/${provider.driver} has no ReplayFixtures test file to run it.`,
46 );
47 }
48 }
49 }
50
51 export const readReplayFixtureTranscript = Effect.fn("readOrchestratorReplayTranscript")(function* (
52 file: URL,
53 ) {
54 return yield* readProviderReplayTranscript(file);
55 }, Effect.provide(NodeServices.layer));
56
57 function normalizeTestError(cause: unknown): Error {
58 return cause instanceof Error ? cause : new Error(String(cause));
59 }
60
61 function transcriptEntriesThroughLabel(
62 transcript: ProviderReplayTranscript,
63 label: string | undefined,
64 ): ProviderReplayTranscript {
65 if (label === undefined) {
66 return transcript;
67 }
68 const entryIndex = transcript.entries.findIndex(
69 (entry) => entry.type === "emit_inbound" && entry.label === label,
70 );
71 if (entryIndex === -1) {
72 throw new Error(`${transcript.scenario} is missing inbound transcript label ${label}.`);
73 }
74 return {
75 ...transcript,
76 entries: transcript.entries.slice(0, entryIndex + 1),
77 };
78 }
79
80 function isStreamingAssistantEvent(event: OrchestrationV2DomainEvent): boolean {
81 switch (event.type) {
82 case "node.updated":
83 return event.payload.kind === "assistant_message" && event.payload.status === "running";
84 case "message.updated":
85 return event.payload.role === "assistant" && event.payload.streaming;
86 case "turn-item.updated":
87 return event.payload.type === "assistant_message" && event.payload.streaming;
88 default:
89 return false;
90 }
91 }
92
93 export interface ReplayFixtureCase {
94 readonly fixtureName: string;
95 readonly buildInput: () => OrchestratorFixtureInput;
96 readonly driver: ProviderOrchestratorReplayVariant;
97 readonly transformTranscript?: (transcript: ProviderReplayTranscript) => ProviderReplayTranscript;
98 /** The instances the ingestor reads snapshots from, such as an endpoint behind a hub. */
99 readonly providerInstances?: Partial<ProviderInstanceRegistry["Service"]>;
100 }
101
102 export interface ReplayFixtureRun<
103 Transcript extends ProviderReplayTranscript,
104 Error,
105 WorkspaceError = never,
106 > extends ReplayFixtureCase {
107 readonly harness: OrchestratorV2ProviderReplayHarness<Transcript, Error>;
108 /** Rewrites recorded workspace paths to the replay workspace, for drivers that record them. */
109 readonly materializeWorkspace?: (
110 transcript: ProviderReplayTranscript,
111 workspace: string,
112 ) => Effect.Effect<ProviderReplayTranscript, WorkspaceError>;
113 }
114
115 const runFixtureProvider = Effect.fn("runOrchestratorReplayFixture")(function* <
116 Transcript extends ProviderReplayTranscript,
117 Error,
118 WorkspaceError,
119 >(input: ReplayFixtureRun<Transcript, Error, WorkspaceError>) {
120 const recordedTranscript = yield* readReplayFixtureTranscript(input.driver.transcriptFile);
121 const rawTranscript = input.transformTranscript?.(recordedTranscript) ?? recordedTranscript;
122 const replayTranscript = materializeReplayTranscriptRuntimeInstructions(
123 transcriptEntriesThroughLabel(rawTranscript, input.driver.transcriptEntriesThroughLabel),
124 { driver: input.driver.driver, model: input.driver.modelSelection.model },
125 );
126 const fixtureInput = input.buildInput();
127 const workspace = yield* checkpointWorkspace(input.fixtureName, fixtureInput.workspaceFiles);
128 const transcript = yield* input.harness.decodeTranscript(
129 input.materializeWorkspace === undefined
130 ? replayTranscript
131 : yield* input.materializeWorkspace(replayTranscript, workspace),
132 );
133 const materialized = yield* materializeFixtureInput({
134 scenario: input.fixtureName,
135 fixtureInput,
136 driver: input.driver.driver,
137 modelSelection: input.driver.modelSelection,
138 }).pipe(Effect.provide(IdAllocator.layer), provideDeterministicTestRuntime);
139 const scenario = {
140 name: `${input.fixtureName}/${input.driver.driver}`,
141 transcript,
142 commands: materialized.commands,
143 steps: materialized.steps,
144 projectionThreadIds: materialized.projectionThreadIds,
145 runtimePolicyOverride: {
146 ...input.driver.runtimePolicyOverride,
147 cwd: workspace,
148 },
149 };
150
151 const result = yield* runOrchestratorV2ProviderReplayScenario(scenario, input.harness, {
152 ...(input.driver.runContinuationWorker === true ? { runContinuationWorker: true } : {}),
153 ...(input.providerInstances === undefined
154 ? {}
155 : { providerInstances: input.providerInstances }),
156 }).pipe(provideDeterministicTestRuntime);
157 input.driver.assertOutput(result, transcript);
158 assertProviderNativeSubagentRootTurns(result);
159 const expectedAbsentWorkspacePaths = input.driver.expectedAbsentWorkspacePaths;
160 if (expectedAbsentWorkspacePaths !== undefined) {
161 yield* Effect.gen(function* () {
162 const fs = yield* FileSystem.FileSystem;
163 const path = yield* Path.Path;
164 for (const relativePath of expectedAbsentWorkspacePaths) {
165 assert.isFalse(
166 yield* fs.exists(path.join(workspace, relativePath)),
167 `${input.fixtureName}/${input.driver.driver} must not create ${relativePath} in the replay workspace`,
168 );
169 }
170 }).pipe(Effect.provide(NodeServices.layer));
171 }
172 assert.isFalse(
173 result.domainEvents.some(isStreamingAssistantEvent),
174 "buffered delivery must not persist streaming assistant artifacts",
175 );
176 const projectionThreadId = materialized.projectionThreadIds[0];
177 assert.isDefined(projectionThreadId);
178 const projection = result.projections.get(projectionThreadId);
179 assert.isDefined(projection);
180 const latestRun = projection.runs.at(-1);
181 // A `set_model` step moves later runs to another model of the same instance.
182 const finalModel = fixtureInput.steps.findLast((step) => step.type === "set_model");
183 assert.deepEqual(
184 latestRun?.modelSelection,
185 finalModel?.type === "set_model"
186 ? { ...input.driver.modelSelection, model: finalModel.model }
187 : input.driver.modelSelection,
188 );
189 if (projection.runs.some((run) => run.status === "completed")) {
190 const threadStartCheckpoint = projection.checkpoints.find(
191 (checkpoint) => checkpoint.ordinalWithinScope === 0 && checkpoint.appRunOrdinal === null,
192 );
193 assert.isDefined(
194 threadStartCheckpoint,
195 "completed threads must retain an addressable thread-start checkpoint",
196 );
197 assert.equal(threadStartCheckpoint.status, "ready");
198 }
199 return result;
200 });
201
202 /** Replays one fixture variant through OrchestratorV2 and runs the shared output checks. */
203 export function runReplayFixture<
204 Transcript extends ProviderReplayTranscript,
205 Error,
206 WorkspaceError = never,
207 >(input: ReplayFixtureRun<Transcript, Error, WorkspaceError>) {
208 return runFixtureProvider(input).pipe(Effect.mapError(normalizeTestError), Effect.scoped);
209 }
210
211 export function findReplayFixture(name: string) {
212 return ORCHESTRATOR_REPLAY_FIXTURES.find((fixture) => fixture.name === name);
213 }
214
215 /**
216 * Registers one test per recorded fixture variant of `driver`. Call it inside the
217 * provider file's `describe`. `run` picks the harness for each variant.
218 */
219 export function itRunsReplayFixtures<E>(
220 driver: string,
221 run: (input: ReplayFixtureCase) => Effect.Effect<unknown, E>,
222 ) {
223 it.effect.each(
224 ORCHESTRATOR_REPLAY_FIXTURES.flatMap((fixture) =>
225 fixture.providers
226 .filter((provider) => provider.driver === driver)
227 .map((provider) => [fixture.name, provider.driver, fixture, provider] as const),
228 ),
229 )("runs %s/%s through OrchestratorV2 using deterministic replay", ([, , fixture, provider]) =>
230 run({ fixtureName: fixture.name, buildInput: fixture.buildInput, driver: provider }),
231 );
232 }