The best way to control Claude Code, Codex and more from any computer or phone

main
ts 579 lines 20.6 KB
Raw
1 import * as NodeServices from "@effect/platform-node/NodeServices";
2 import { it } from "@effect/vitest";
3 import { ClaudeSettings, ProviderInstanceId } from "@t3tools/contracts";
4 import * as HostProcess from "@t3tools/shared/HostProcess";
5 import { createModelSelection } from "@t3tools/shared/model";
6 import * as Effect from "effect/Effect";
7 import * as FileSystem from "effect/FileSystem";
8 import * as Layer from "effect/Layer";
9 import * as Path from "effect/Path";
10 import * as Schema from "effect/Schema";
11 import { expect } from "vite-plus/test";
12
13 import * as ServerConfig from "../config.ts";
14 import { processProbeSource } from "../testUtils/processProbe.ts";
15 import {
16 SYNTHETIC_CLAUDE_CAPABLE_MODEL,
17 SYNTHETIC_CLAUDE_COLLIDING_ALIAS,
18 SYNTHETIC_CLAUDE_MODEL_CATALOG,
19 SYNTHETIC_CLAUDE_STANDARD_MODEL,
20 } from "../provider/ClaudeModelCatalog.testFixtures.ts";
21 import * as TextGeneration from "./TextGeneration.ts";
22 import { sanitizeThreadTitle } from "@t3tools/provider-core/server/textGenerationUtils";
23 import { makeClaudeTextGeneration } from "./ClaudeTextGeneration.ts";
24 import { writeFakeCli } from "@t3tools/provider-testing/fakeCli";
25 const decodeClaudeSettings = Schema.decodeSync(ClaudeSettings);
26
27 const layerClaudeTextGenerationTest = ServerConfig.ServerConfig.layerTest(process.cwd(), {
28 prefix: "t3code-claude-text-generation-test-",
29 }).pipe(Layer.provideMerge(NodeServices.layer));
30
31 // The stub behaviour lives in Node so the same implementation runs on Windows,
32 // where a shebang file is not executable and would fall through to the real
33 // Claude CLI on PATH; `writeFakeCli` picks the launcher shape per host.
34 function makeFakeClaudeBinary(dir: string, lifecyclePort?: number) {
35 return Effect.gen(function* () {
36 const path = yield* Path.Path;
37 const fs = yield* FileSystem.FileSystem;
38 const platform = yield* HostProcess.Platform;
39 const binDir = path.join(dir, "bin");
40 const fixturePath = yield* path.fromFileUrl(
41 new URL("./testing/ClaudeTextGeneration.fixture.mjs", import.meta.url),
42 );
43 const source = yield* fs.readFileString(fixturePath);
44 writeFakeCli({
45 directory: binDir,
46 name: "claude",
47 platform,
48 source:
49 lifecyclePort === undefined
50 ? source
51 : `process.on("SIGTERM", () => {}); ${processProbeSource(lifecyclePort)}`,
52 });
53 return binDir;
54 });
55 }
56
57 function withFakeClaudeEnv<A, E, R>(
58 input: {
59 output: string;
60 rejectsNoSessionPersistence?: boolean;
61 lifecyclePort?: number;
62 exitCode?: number;
63 stderr?: string;
64 argsMustContain?: string;
65 expectedTools?: string;
66 argsMustNotContain?: string;
67 stdinMustContain?: string;
68 rejectsImages?: boolean;
69 configDirMustBe?: string;
70 cwdMustNotBe?: string;
71 claudeConfig?: Partial<ClaudeSettings>;
72 },
73 effectFn: (textGeneration: TextGeneration.TextGeneration["Service"]) => Effect.Effect<A, E, R>,
74 ) {
75 return Effect.gen(function* () {
76 const fs = yield* FileSystem.FileSystem;
77 const tempDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3code-claude-text-" });
78 const binDir = yield* makeFakeClaudeBinary(tempDir, input.lifecyclePort);
79 const pathDelimiter = (yield* HostProcess.isWindows) ? ";" : ":";
80 const hostEnv = yield* HostProcess.Environment;
81 const fakeClaudeEnv: Record<string, string | undefined> = {
82 T3_FAKE_CLAUDE_OUTPUT: input.output,
83 T3_FAKE_CLAUDE_REJECTS_NO_SESSION_PERSISTENCE: input.rejectsNoSessionPersistence
84 ? "1"
85 : undefined,
86 T3_FAKE_CLAUDE_EXIT_CODE: input.exitCode === undefined ? undefined : String(input.exitCode),
87 T3_FAKE_CLAUDE_STDERR: input.stderr,
88 T3_FAKE_CLAUDE_EXPECTED_TOOLS: input.expectedTools,
89 T3_FAKE_CLAUDE_ARGS_MUST_CONTAIN: input.argsMustContain,
90 T3_FAKE_CLAUDE_ARGS_MUST_NOT_CONTAIN: input.argsMustNotContain,
91 T3_FAKE_CLAUDE_STDIN_MUST_CONTAIN: input.stdinMustContain,
92 T3_FAKE_CLAUDE_REJECTS_IMAGES: input.rejectsImages ? "1" : undefined,
93 T3_FAKE_CLAUDE_CWD_MUST_NOT_BE: input.cwdMustNotBe,
94 T3_FAKE_CLAUDE_CONFIG_DIR_MUST_BE: input.configDirMustBe,
95 };
96 const environment: NodeJS.ProcessEnv = {
97 ...Object.fromEntries(
98 Object.entries(hostEnv).filter(([name]) => !name.startsWith("T3_FAKE_CLAUDE_")),
99 ),
100 ...Object.fromEntries(
101 Object.entries(fakeClaudeEnv).filter(([, value]) => value !== undefined),
102 ),
103 PATH: `${binDir}${pathDelimiter}${hostEnv.PATH ?? ""}`,
104 };
105
106 const config = decodeClaudeSettings(input.claudeConfig ?? {});
107 const textGeneration = yield* makeClaudeTextGeneration(
108 config,
109 environment,
110 Effect.succeed(SYNTHETIC_CLAUDE_MODEL_CATALOG),
111 );
112 return yield* effectFn(textGeneration);
113 }).pipe(Effect.scoped);
114 }
115
116 it.layer(layerClaudeTextGenerationTest)("ClaudeTextGeneration", (it) => {
117 it.effect("loads only user settings when generating review guides", () =>
118 withFakeClaudeEnv(
119 {
120 output: JSON.stringify({ structured_output: { stops: [] } }),
121 argsMustContain: "--setting-sources user",
122 expectedTools: "Read,Grep,Glob,Bash",
123 },
124 (textGeneration) =>
125 textGeneration.generateReviewGuide({
126 cwd: process.cwd(),
127 number: 12,
128 title: "Review a change",
129 description: "",
130 baseBranch: "main",
131 baseCommit: "base",
132 settledReview: "",
133 modelSelection: createModelSelection(
134 ProviderInstanceId.make("claudeAgent"),
135 SYNTHETIC_CLAUDE_CAPABLE_MODEL,
136 ),
137 }),
138 ),
139 );
140
141 it.effect(
142 "keeps canonical built-in capabilities when a custom model collides with its alias",
143 () =>
144 withFakeClaudeEnv(
145 {
146 output: JSON.stringify({
147 structured_output: {
148 title: "Improve orchestration flow",
149 body: "Body",
150 },
151 }),
152 argsMustContain: `--model ${SYNTHETIC_CLAUDE_CAPABLE_MODEL}[expanded] --effort max --settings {"disableAllHooks":true,"fastMode":true}`,
153 claudeConfig: { customModels: [SYNTHETIC_CLAUDE_COLLIDING_ALIAS] },
154 },
155 (textGeneration) =>
156 Effect.gen(function* () {
157 const generated = yield* textGeneration.generatePrContent({
158 cwd: process.cwd(),
159 baseBranch: "main",
160 headBranch: "feature/claude-effect",
161 commitSummary: "Improve orchestration",
162 diffSummary: "1 file changed",
163 diffPatch: "diff --git a/README.md b/README.md",
164 modelSelection: {
165 ...createModelSelection(
166 ProviderInstanceId.make("claudeAgent"),
167 SYNTHETIC_CLAUDE_CAPABLE_MODEL,
168 [
169 { id: "effort", value: "max" },
170 { id: "fastMode", value: true },
171 ],
172 ),
173 },
174 });
175
176 expect(generated.title).toBe("Improve orchestration flow");
177 }),
178 ),
179 );
180
181 it.effect(
182 "generates thread titles outside the project with tools, skills, and hooks disabled",
183 () =>
184 withFakeClaudeEnv(
185 {
186 output: JSON.stringify({
187 structured_output: {
188 title:
189 ' "Reconnect failures after restart because the session state does not recover" ',
190 },
191 }),
192 cwdMustNotBe: process.cwd(),
193 stdinMustContain: "/call-script",
194 },
195 (textGeneration) =>
196 Effect.gen(function* () {
197 const generated = yield* textGeneration.generateThreadTitle({
198 cwd: process.cwd(),
199 message: "/call-script",
200 modelSelection: {
201 instanceId: ProviderInstanceId.make("claudeAgent"),
202 model: SYNTHETIC_CLAUDE_STANDARD_MODEL,
203 },
204 });
205
206 expect(generated.title).toBe(
207 sanitizeThreadTitle(
208 '"Reconnect failures after restart because the session state does not recover"',
209 ),
210 );
211 }),
212 ),
213 );
214
215 it.effect("sends thread title instructions to the provider prompt", () =>
216 withFakeClaudeEnv(
217 {
218 output: JSON.stringify({ structured_output: { title: "Reconnect failures" } }),
219 stdinMustContain: "Title instructions from the user:\nUse two to four words.",
220 },
221 (textGeneration) =>
222 textGeneration.generateThreadTitle({
223 cwd: process.cwd(),
224 message: "Investigate reconnect failures",
225 instructions: "Use two to four words.",
226 modelSelection: {
227 instanceId: ProviderInstanceId.make("claudeAgent"),
228 model: SYNTHETIC_CLAUDE_STANDARD_MODEL,
229 },
230 }),
231 ),
232 );
233
234 it.effect("sends image attachments to Claude, not only their metadata", () =>
235 withFakeClaudeEnv(
236 {
237 output: JSON.stringify({ structured_output: { branch: "fix/login-button" } }),
238 stdinMustContain: Buffer.from("hello").toString("base64"),
239 },
240 (textGeneration) =>
241 Effect.gen(function* () {
242 const fs = yield* FileSystem.FileSystem;
243 const path = yield* Path.Path;
244 const { attachmentsDir } = yield* ServerConfig.ServerConfig;
245 const attachmentId = "thread-1-claude-attachment";
246 const imagePath = path.join(attachmentsDir, `${attachmentId}.png`);
247 yield* fs.makeDirectory(attachmentsDir, { recursive: true });
248 yield* fs.writeFile(imagePath, Buffer.from("hello"));
249
250 yield* textGeneration
251 .generateBranchName({
252 cwd: process.cwd(),
253 message: "fix this",
254 attachments: [
255 {
256 type: "image",
257 id: attachmentId,
258 name: "screen.png",
259 mimeType: "image/png",
260 sizeBytes: 5,
261 },
262 ],
263 modelSelection: {
264 instanceId: ProviderInstanceId.make("claudeAgent"),
265 model: SYNTHETIC_CLAUDE_STANDARD_MODEL,
266 },
267 })
268 .pipe(Effect.ensuring(fs.remove(imagePath).pipe(Effect.ignore)));
269 }),
270 ),
271 );
272
273 it.effect("still generates text when a text-only model rejects image attachments", () =>
274 withFakeClaudeEnv(
275 {
276 output: JSON.stringify({ structured_output: { branch: "fix/login-button" } }),
277 rejectsImages: true,
278 },
279 (textGeneration) =>
280 Effect.gen(function* () {
281 const fs = yield* FileSystem.FileSystem;
282 const path = yield* Path.Path;
283 const { attachmentsDir } = yield* ServerConfig.ServerConfig;
284 const attachmentId = "thread-1-claude-text-only-attachment";
285 const imagePath = path.join(attachmentsDir, `${attachmentId}.png`);
286 yield* fs.makeDirectory(attachmentsDir, { recursive: true });
287 yield* fs.writeFile(imagePath, Buffer.from("hello"));
288
289 const result = yield* textGeneration
290 .generateBranchName({
291 cwd: process.cwd(),
292 message: "fix this",
293 attachments: [
294 {
295 type: "image",
296 id: attachmentId,
297 name: "screen.png",
298 mimeType: "image/png",
299 sizeBytes: 5,
300 },
301 ],
302 modelSelection: {
303 instanceId: ProviderInstanceId.make("claudeAgent"),
304 model: SYNTHETIC_CLAUDE_STANDARD_MODEL,
305 },
306 })
307 .pipe(Effect.ensuring(fs.remove(imagePath).pipe(Effect.ignore)));
308
309 expect(result.branch).toBe("fix/login-button");
310 }),
311 ),
312 );
313
314 it.effect("runs Claude text generation with the configured CLAUDE_CONFIG_DIR", () =>
315 Effect.gen(function* () {
316 const path = yield* Path.Path;
317 const claudeConfigDir = path.join(process.cwd(), ".claude-work-test");
318 return yield* withFakeClaudeEnv(
319 {
320 output: JSON.stringify({
321 structured_output: {
322 title: "Use Claude home",
323 },
324 }),
325 configDirMustBe: claudeConfigDir,
326 claudeConfig: { homePath: claudeConfigDir },
327 },
328 (textGeneration) =>
329 Effect.gen(function* () {
330 const generated = yield* textGeneration.generateThreadTitle({
331 cwd: process.cwd(),
332 message: "thread title",
333 modelSelection: {
334 instanceId: ProviderInstanceId.make("claudeAgent"),
335 model: SYNTHETIC_CLAUDE_STANDARD_MODEL,
336 },
337 });
338
339 expect(generated.title).toBe(sanitizeThreadTitle("Use Claude home"));
340 }),
341 );
342 }),
343 );
344
345 it.effect("passes --settings from the instance launch args to Claude text generation", () =>
346 withFakeClaudeEnv(
347 {
348 output: JSON.stringify({
349 structured_output: {
350 title: "Use instance settings",
351 },
352 }),
353 argsMustContain: `"env":{"ANTHROPIC_VERTEX_PROJECT_ID":"vertex-project-a"}`,
354 claudeConfig: {
355 launchArgs: `--settings '{"env":{"ANTHROPIC_VERTEX_PROJECT_ID":"vertex-project-a"}}'`,
356 },
357 },
358 (textGeneration) =>
359 Effect.gen(function* () {
360 const generated = yield* textGeneration.generateThreadTitle({
361 cwd: process.cwd(),
362 message: "thread title",
363 modelSelection: {
364 instanceId: ProviderInstanceId.make("claudeAgent"),
365 model: SYNTHETIC_CLAUDE_STANDARD_MODEL,
366 },
367 });
368
369 expect(generated.title).toBe(sanitizeThreadTitle("Use instance settings"));
370 }),
371 ),
372 );
373
374 it.effect("retries without --no-session-persistence when the Claude CLI rejects it", () =>
375 withFakeClaudeEnv(
376 {
377 output: JSON.stringify({ structured_output: { title: "Reconnect failures" } }),
378 rejectsNoSessionPersistence: true,
379 },
380 (textGeneration) =>
381 Effect.gen(function* () {
382 const generated = yield* textGeneration.generateThreadTitle({
383 cwd: process.cwd(),
384 message: "Investigate reconnect failures",
385 modelSelection: {
386 instanceId: ProviderInstanceId.make("claudeAgent"),
387 model: SYNTHETIC_CLAUDE_STANDARD_MODEL,
388 },
389 });
390
391 expect(generated.title).toBe("Reconnect failures");
392 }),
393 ),
394 );
395
396 it.effect.each([
397 { verbose: false, label: "normal" },
398 { verbose: true, label: "verbose" },
399 ])("unwraps a JSON title in $label Claude output", ({ verbose }) => {
400 const result = {
401 type: "result",
402 structured_output: { title: '{"title": "Refresh ev-stg APP ASG instances"}' },
403 };
404 return withFakeClaudeEnv(
405 { output: JSON.stringify(verbose ? [result] : result) },
406 (textGeneration) =>
407 Effect.gen(function* () {
408 const generated = yield* textGeneration.generateThreadTitle({
409 cwd: process.cwd(),
410 message: "Refresh ev-stg APP ASG instances",
411 modelSelection: {
412 instanceId: ProviderInstanceId.make("claudeAgent"),
413 model: SYNTHETIC_CLAUDE_STANDARD_MODEL,
414 },
415 });
416
417 expect(generated.title).toBe("Refresh ev-stg APP ASG instances");
418 }),
419 );
420 });
421
422 it.effect.each([
423 { previousTitle: undefined, label: "generating" },
424 { previousTitle: "Old thread title", label: "regenerating" },
425 ])("reads the result from verbose Claude output when $label a title", ({ previousTitle }) =>
426 withFakeClaudeEnv(
427 {
428 output: JSON.stringify([
429 { type: "system", subtype: "init" },
430 { type: "assistant", message: { content: [] } },
431 { type: "user", message: { content: [] } },
432 { type: "rate_limit_event" },
433 {
434 type: "result",
435 subtype: "success",
436 result: '{"title":"Refresh ev-stg APP ASG Instances"}',
437 structured_output: { title: "Refresh ev-stg APP ASG Instances" },
438 },
439 ]),
440 },
441 (textGeneration) =>
442 Effect.gen(function* () {
443 const generated = yield* textGeneration.generateThreadTitle({
444 cwd: process.cwd(),
445 message: "Refresh ev-stg APP ASG instances",
446 previousTitle,
447 modelSelection: {
448 instanceId: ProviderInstanceId.make("claudeAgent"),
449 model: SYNTHETIC_CLAUDE_STANDARD_MODEL,
450 },
451 });
452
453 expect(generated.title).toBe("Refresh ev-stg APP ASG Instances");
454 }),
455 ),
456 );
457
458 it.effect.each([
459 ["empty message array", []],
460 ["missing result", [{ type: "assistant", structured_output: { title: "Not a result" } }]],
461 ["invalid title", [{ type: "result", structured_output: { title: 42 } }]],
462 [
463 "final result without structured output",
464 [
465 { type: "result", structured_output: { title: "Earlier result" } },
466 { type: "result", subtype: "error_max_structured_output_retries" },
467 ],
468 ],
469 ] as const)("rejects verbose Claude output with %s", ([, output]) =>
470 withFakeClaudeEnv({ output: JSON.stringify(output) }, (textGeneration) =>
471 Effect.gen(function* () {
472 const error = yield* Effect.flip(
473 textGeneration.generateThreadTitle({
474 cwd: process.cwd(),
475 message: "Name this thread",
476 modelSelection: {
477 instanceId: ProviderInstanceId.make("claudeAgent"),
478 model: SYNTHETIC_CLAUDE_STANDARD_MODEL,
479 },
480 }),
481 );
482
483 expect(error._tag).toBe("TextGenerationError");
484 }),
485 ),
486 );
487
488 const apiError = "API Error: 400 Model blocked by guardrail: 4 endpoints excluded";
489 const wrapperNotice = "Using the OpenRouter credential from the global credential file.";
490 const structuredOutputError = "Failed to provide valid structured output after 5 attempts";
491 const structuredOutputFailure = {
492 type: "result",
493 subtype: "error_max_structured_output_retries",
494 is_error: true,
495 errors: [structuredOutputError],
496 };
497 it.effect.each([
498 [
499 "the errors list of a result with no result text",
500 JSON.stringify(structuredOutputFailure),
501 structuredOutputError,
502 ],
503 [
504 "the errors list from verbose output",
505 JSON.stringify([{ type: "system", subtype: "init" }, structuredOutputFailure]),
506 structuredOutputError,
507 ],
508 [
509 "the errors list without CLI telemetry",
510 JSON.stringify({
511 ...structuredOutputFailure,
512 subtype: "error_during_execution",
513 errors: ["[ede_diagnostic] result_type=user last_content_type=n/a", apiError],
514 }),
515 apiError,
516 ],
517 [
518 "the stdout error over a stderr notice",
519 JSON.stringify({ is_error: true, api_error_status: 400, result: apiError }),
520 apiError,
521 ],
522 [
523 "the verbose stdout error over a stderr notice",
524 JSON.stringify([
525 { type: "system", subtype: "init" },
526 { type: "result", is_error: true, result: apiError },
527 { type: "result", is_error: true, result: " " },
528 ]),
529 apiError,
530 ],
531 ["stderr when stdout has no error", "not json", wrapperNotice],
532 [
533 "a stdout error naming --no-session-persistence without retrying",
534 JSON.stringify({
535 is_error: true,
536 result: "error: unknown option '--no-session-persistence'",
537 }),
538 "error: unknown option '--no-session-persistence'",
539 ],
540 ] as const)("reports %s when the Claude CLI fails", ([, output, expected]) =>
541 withFakeClaudeEnv({ output, stderr: wrapperNotice, exitCode: 1 }, (textGeneration) =>
542 Effect.gen(function* () {
543 const error = yield* Effect.flip(
544 textGeneration.generateThreadTitle({
545 cwd: process.cwd(),
546 message: "Name this thread",
547 modelSelection: {
548 instanceId: ProviderInstanceId.make("claudeAgent"),
549 model: SYNTHETIC_CLAUDE_STANDARD_MODEL,
550 },
551 }),
552 );
553
554 expect(error.detail).toBe(`Claude CLI command failed: ${expected}`);
555 }),
556 ),
557 );
558
559 it.effect("does not report the raw result JSON when a failed run writes nothing to stderr", () =>
560 withFakeClaudeEnv(
561 { output: JSON.stringify(structuredOutputFailure), exitCode: 1 },
562 (textGeneration) =>
563 Effect.gen(function* () {
564 const error = yield* Effect.flip(
565 textGeneration.generateThreadTitle({
566 cwd: process.cwd(),
567 message: "Name this thread",
568 modelSelection: {
569 instanceId: ProviderInstanceId.make("claudeAgent"),
570 model: SYNTHETIC_CLAUDE_STANDARD_MODEL,
571 },
572 }),
573 );
574
575 expect(error.detail).toBe(`Claude CLI command failed: ${structuredOutputError}`);
576 }),
577 ),
578 );
579 });