The best way to control Claude Code, Codex and more from any computer or phone

main
ts 409 lines 16.8 KB
Raw
1 import * as NodeServices from "@effect/platform-node/NodeServices";
2 import { assert, describe, it } from "@effect/vitest";
3 import {
4 CommandId,
5 isProviderNativeSubagentThread,
6 MessageId,
7 ProviderDriverKind,
8 type ProviderReplayEntry,
9 type ServerProvider,
10 UsageLimitSourceId,
11 type UsageLimitSourceSnapshot,
12 } from "@t3tools/contracts";
13 import * as Effect from "effect/Effect";
14 import * as Layer from "effect/Layer";
15
16 import { ClaudeOrchestratorReplayHarness } from "../Adapters/ClaudeAdapterV2.testkit.ts";
17 import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
18 import * as Orchestrator from "../Orchestrator.ts";
19 import { userFacingDispatchErrorMessage } from "../UserFacingErrors.ts";
20 import * as UsageLimitSources from "../../usage/UsageLimitSources.ts";
21 import { claudeUsageLimitFailure } from "./fixtures/usage_limit/claude_output.ts";
22 import { provideDeterministicTestRuntime } from "./DeterministicRuntime.ts";
23 import { subagentInput } from "./fixtures/subagent/input.ts";
24 import { runOrchestratorV2Scenario } from "./OrchestratorScenario.ts";
25 import * as ProviderReplayHarness from "./ProviderReplayHarness.ts";
26 import { materializeReplayTranscriptRuntimeInstructions } from "./ReplayRuntimeInstructions.ts";
27 import {
28 CLAUDE_MODEL_SELECTION,
29 materializeFixtureInput,
30 projectionFor,
31 } from "./fixtures/shared.ts";
32 import {
33 findReplayFixture,
34 itRunsReplayFixtures,
35 runReplayFixture,
36 type ReplayFixtureCase,
37 } from "./ProviderReplayFixtures.ts";
38 import { readProviderReplayTranscript } from "@t3tools/provider-testing/replayTranscript";
39
40 const readTranscript = Effect.fn("readClaudeReplayFixture")(function* (file: URL) {
41 return yield* readProviderReplayTranscript(file);
42 }, Effect.provide(NodeServices.layer));
43
44 function readClaudeTranscriptFixture(path: string) {
45 return readTranscript(new URL(`./fixtures/${path}/claude_transcript.ndjson`, import.meta.url));
46 }
47
48 function runClaudeReplayFixture(input: ReplayFixtureCase) {
49 return runReplayFixture({ ...input, harness: ClaudeOrchestratorReplayHarness });
50 }
51
52 describe("Claude Agent SDK replay fixtures", () => {
53 itRunsReplayFixtures("claudeAgent", runClaudeReplayFixture);
54
55 it.effect("refuses messages to a native subagent thread without touching it", () =>
56 Effect.gen(function* () {
57 const raw = yield* readClaudeTranscriptFixture("subagent");
58 const transcript = yield* ClaudeOrchestratorReplayHarness.decodeTranscript(
59 materializeReplayTranscriptRuntimeInstructions(raw, {
60 driver: ProviderDriverKind.make("claudeAgent"),
61 model: CLAUDE_MODEL_SELECTION.model,
62 }),
63 );
64 const materialized = yield* materializeFixtureInput({
65 scenario: "subagent",
66 fixtureInput: subagentInput(),
67 driver: ProviderDriverKind.make("claudeAgent"),
68 modelSelection: CLAUDE_MODEL_SELECTION,
69 }).pipe(Effect.provide(IdAllocator.layer), provideDeterministicTestRuntime);
70 const scenario = {
71 name: "subagent/claudeAgent:read-only-child",
72 transcript,
73 commands: materialized.commands,
74 steps: materialized.steps,
75 projectionThreadIds: materialized.projectionThreadIds,
76 };
77 yield* Effect.gen(function* () {
78 const result = yield* runOrchestratorV2Scenario(scenario);
79 const orchestrator = yield* Orchestrator.OrchestratorV2;
80 const child = [...result.projections.values()].find((projection) =>
81 isProviderNativeSubagentThread(projection.thread),
82 );
83 assert.isDefined(child);
84 const before = yield* orchestrator.getThreadEventSequence(child.thread.id);
85
86 const refused = yield* orchestrator
87 .dispatch({
88 type: "message.dispatch",
89 createdBy: "user",
90 creationSource: "web",
91 commandId: CommandId.make("command:subagent:message-native-child"),
92 threadId: child.thread.id,
93 messageId: MessageId.make("message:subagent:message-native-child"),
94 text: "Also check the tests.",
95 attachments: [],
96 dispatchMode: { type: "start_immediately" },
97 })
98 .pipe(Effect.flip);
99 assert.equal(refused._tag, "OrchestratorSubagentThreadReadOnlyError");
100 assert.equal(
101 userFacingDispatchErrorMessage(refused),
102 "This subagent is run by its provider and cannot take messages. Message the parent thread instead.",
103 );
104 const refusedContinuation = yield* orchestrator
105 .dispatch({
106 type: "thread.metadata.update",
107 commandId: CommandId.make("command:subagent:continue-native-child"),
108 threadId: child.thread.id,
109 title: "Renamed",
110 continuation: {
111 messageId: MessageId.make("message:subagent:continue-native-child"),
112 text: "Also check the tests.",
113 },
114 })
115 .pipe(Effect.flip);
116 assert.equal(refusedContinuation._tag, "OrchestratorSubagentThreadReadOnlyError");
117 assert.equal(yield* orchestrator.getThreadEventSequence(child.thread.id), before);
118 const after = yield* orchestrator.getThreadProjection(child.thread.id);
119 assert.lengthOf(after.runs, 0);
120 assert.deepEqual(after.messages, child.messages);
121 }).pipe(
122 Effect.provide(
123 ProviderReplayHarness.layerProviderReplay(scenario, ClaudeOrchestratorReplayHarness),
124 ),
125 provideDeterministicTestRuntime,
126 Effect.scoped,
127 );
128 }),
129 );
130
131 it.effect("takes the reset of the hub accounts that serve the model the hub refused", () =>
132 Effect.gen(function* () {
133 const fixture = findReplayFixture("usage_limit")!;
134 const driver = fixture.providers[0]!;
135 const endpoint = "http://127.0.0.1:18319";
136 const account = (id: string, model: string, usedPercent: number, resetsAt: string) => ({
137 id,
138 driver: ProviderDriverKind.make("claudeAgent"),
139 models: [model],
140 usageLimits: {
141 checkedAt: resetsAt,
142 windows: [
143 { id: "five_hour", kind: "session" as const, label: "Session", usedPercent, resetsAt },
144 ],
145 },
146 });
147 const hub = (id: string, origin: string, accounts: ReadonlyArray<object>) =>
148 ({
149 id: UsageLimitSourceId.make(id),
150 kind: "cliproxy",
151 label: id,
152 origin,
153 checkedAt: "",
154 accounts,
155 }) as UsageLimitSourceSnapshot;
156 yield* runReplayFixture({
157 fixtureName: fixture.name,
158 buildInput: fixture.buildInput,
159 driver: {
160 ...driver,
161 assertOutput: (result, transcript) =>
162 assert.equal(
163 claudeUsageLimitFailure(result, transcript).resetAt,
164 "2033-05-19T07:20:00.000Z",
165 ),
166 },
167 harness: ClaudeOrchestratorReplayHarness,
168 providerInstances: {
169 getInstance: () =>
170 Effect.succeed({
171 snapshot: {
172 getSnapshot: Effect.succeed({
173 auth: { status: "unknown", type: "customEndpoint", endpoint },
174 } as ServerProvider),
175 },
176 } as never),
177 },
178 }).pipe(
179 Effect.provide(
180 UsageLimitSources.layerResetFallback.pipe(
181 Layer.provide(
182 Layer.mock(UsageLimitSources.UsageLimitSources)({
183 current: Effect.succeed([
184 hub("home", endpoint, [
185 account("sonnet", driver.modelSelection.model, 100, "2033-05-19T07:20:00.000Z"),
186 // regression: #30778 an account the hub does not route the model to has no say
187 account("opus", "claude-opus-4-8", 10, "2033-05-19T05:00:00.000Z"),
188 ]),
189 hub("elsewhere", "http://127.0.0.1:9999", [
190 account("other", driver.modelSelection.model, 100, "2033-05-18T00:00:00.000Z"),
191 ]),
192 ]),
193 refresh: Effect.void,
194 }),
195 ),
196 ),
197 ),
198 );
199 }),
200 );
201
202 const afterRootFixture = findReplayFixture("claude_background_subagent_after_root");
203 const afterRootProvider = afterRootFixture?.providers[0];
204 if (afterRootFixture !== undefined && afterRootProvider !== undefined) {
205 const OPEN_TOOL_ID = "toolu_01QDa5jV5g1H9h6QyDfeJohD";
206 const SUBAGENT_TASK_ID = "a2995bfced8019363";
207
208 const moveOpenToolBeforeRootResult = (
209 move: boolean,
210 entries: ReadonlyArray<ProviderReplayEntry>,
211 ): ReadonlyArray<ProviderReplayEntry> => {
212 if (!move) return entries;
213 const toolIndex = entries.findIndex(
214 (entry) =>
215 entry.type === "emit_inbound" &&
216 (entry.frame as Record<string, any>).message?.content?.[0]?.id === OPEN_TOOL_ID,
217 );
218 const resultIndex = entries.findIndex(
219 (entry) => entry.type === "emit_inbound" && entry.label === "result",
220 );
221 if (toolIndex < 0 || resultIndex < 0) throw new Error("transcript shape changed");
222 const toolEntry = entries[toolIndex]!;
223 const rest = entries.filter((_, index) => index !== toolIndex);
224 return [...rest.slice(0, resultIndex), toolEntry, ...rest.slice(resultIndex)];
225 };
226 // A subagent's foreground Bash run during the root's turn is tailed like a
227 // root call's, so the child's command row streams live.
228 const SUBAGENT_BASH_ID = "toolu_01JMN3SSQCHzUTNua9AWdkiH";
229 it.effect("streams a subagent's foreground Bash output live during the root turn", () =>
230 runClaudeReplayFixture({
231 fixtureName: afterRootFixture.name,
232 buildInput: afterRootFixture.buildInput,
233 driver: {
234 ...afterRootProvider,
235 assertOutput: (result) => {
236 const childThreadId = projectionFor(result, afterRootFixture.name).subagents[0]
237 ?.childThreadId;
238 assert.exists(childThreadId);
239 const command = result.projections
240 .get(childThreadId)
241 ?.turnItems.find(
242 (item) =>
243 item.type === "command_execution" &&
244 JSON.stringify(item.input).includes("SUB_DONE_1"),
245 );
246 assert.exists(command);
247 const streamed = result.commandOutput?.get(command.id);
248 assert.exists(streamed, "the subagent's command must stream its output");
249 assert.include(streamed.output.text, "SUB_DONE_1");
250 },
251 },
252 transformTranscript: (transcript) => {
253 const isBashFrame = (entry: ProviderReplayEntry) => {
254 if (entry.type !== "emit_inbound") return false;
255 const frame = entry.frame as Record<string, any>;
256 return (
257 frame.message?.content?.[0]?.id === SUBAGENT_BASH_ID ||
258 frame.message?.content?.[0]?.tool_use_id === SUBAGENT_BASH_ID ||
259 (frame.type === "system" && frame.tool_use_id === SUBAGENT_BASH_ID)
260 );
261 };
262 // Recorded without the structured result the replay writes output files from.
263 const bashFrames = transcript.entries.filter(isBashFrame).map((entry) => {
264 const frame = (entry as { readonly frame: Record<string, any> }).frame;
265 return frame.type === "user"
266 ? {
267 ...entry,
268 frame: { ...frame, tool_use_result: { stdout: "SUB_DONE_1\n", stderr: "" } },
269 }
270 : entry;
271 });
272 const rest = transcript.entries.filter((entry) => !isBashFrame(entry));
273 const resultIndex = rest.findIndex(
274 (entry) => entry.type === "emit_inbound" && entry.label === "result",
275 );
276 assert.lengthOf(bashFrames, 4, "transcript shape changed");
277 assert.isAtLeast(resultIndex, 0, "transcript shape changed");
278 return {
279 ...transcript,
280 entries: [...rest.slice(0, resultIndex), ...bashFrames, ...rest.slice(resultIndex)],
281 };
282 },
283 }),
284 );
285
286 // regression: #16486 a background subagent's tool left open while its parent is idle never settled
287 it.effect.each([
288 ["WebFetch", "while root is idle", "stopped"],
289 ["Write", "before root settles", "failed"],
290 ] as const)(
291 "ends a background subagent's open %s (opened %s) when its notification is %s",
292 ([tool, opens, notificationStatus]) =>
293 runClaudeReplayFixture({
294 fixtureName: afterRootFixture.name,
295 buildInput: afterRootFixture.buildInput,
296 driver: {
297 ...afterRootProvider,
298 assertOutput: (result) => {
299 const parent = projectionFor(result, afterRootFixture.name);
300 assert.deepEqual(
301 parent.runs.map((run) => run.status),
302 ["completed", "completed"],
303 );
304 const subagent = parent.subagents[0];
305 assert.equal(
306 subagent?.status,
307 notificationStatus === "stopped" ? "cancelled" : "failed",
308 );
309 const childThreadId = subagent?.childThreadId;
310 assert.exists(childThreadId);
311 const child = result.projections.get(childThreadId);
312 assert.exists(child);
313 const fetches = child.turnItems.filter(
314 (item) => item.type === (tool === "WebFetch" ? "web_search" : "file_change"),
315 );
316 assert.lengthOf(fetches, 1, `the ${tool} reached the child thread`);
317 assert.deepEqual(
318 {
319 openTool: fetches.map((item) => item.status),
320 openChildNodes: child.nodes
321 .filter(
322 (node) =>
323 node.status === "running" ||
324 node.status === "pending" ||
325 node.status === "waiting",
326 )
327 .map((node) => `${node.kind}:${node.status}`),
328 },
329 {
330 openTool: [notificationStatus === "stopped" ? "interrupted" : "failed"],
331 openChildNodes: [],
332 },
333 );
334 },
335 },
336 transformTranscript: (transcript) => ({
337 ...transcript,
338 entries: moveOpenToolBeforeRootResult(
339 opens === "before root settles",
340 transcript.entries.flatMap((entry): ReadonlyArray<ProviderReplayEntry> => {
341 if (entry.type !== "emit_inbound") return [entry];
342 const frame = entry.frame as Record<string, any>;
343 if (
344 frame.type === "assistant" &&
345 frame.message?.content?.[0]?.id === OPEN_TOOL_ID
346 ) {
347 return [
348 {
349 ...entry,
350 frame: {
351 ...frame,
352 message: {
353 ...frame.message,
354 content: [
355 {
356 ...frame.message.content[0],
357 name: tool,
358 input:
359 tool === "WebFetch"
360 ? { url: "https://example.com", prompt: "Summarize" }
361 : { file_path: "notes.txt", content: "SUB_DONE_2" },
362 },
363 ],
364 },
365 },
366 },
367 ];
368 }
369 if (
370 (frame.type === "system" &&
371 (frame.subtype === "task_started" || frame.subtype === "task_notification") &&
372 frame.tool_use_id === OPEN_TOOL_ID) ||
373 (frame.type === "user" &&
374 frame.message?.content?.[0]?.tool_use_id === OPEN_TOOL_ID) ||
375 (frame.type === "assistant" &&
376 frame.parent_tool_use_id !== null &&
377 frame.message?.content?.[0]?.text === "SUB_FINAL_REPORT")
378 ) {
379 return [];
380 }
381 if (frame.type === "system" && frame.subtype === "task_updated") {
382 return [
383 {
384 ...entry,
385 frame: {
386 ...frame,
387 patch: {
388 ...frame.patch,
389 status: notificationStatus === "stopped" ? "killed" : "failed",
390 },
391 },
392 },
393 ];
394 }
395 if (
396 frame.type === "system" &&
397 frame.subtype === "task_notification" &&
398 frame.task_id === SUBAGENT_TASK_ID
399 ) {
400 return [{ ...entry, frame: { ...frame, status: notificationStatus } }];
401 }
402 return [entry];
403 }),
404 ),
405 }),
406 }),
407 );
408 }
409 });