The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 03:37 UTC
| 1 | #!/usr/bin/env node |
| 2 | import * as NodeUtil from "node:util"; |
| 3 | import * as NodeServices from "@effect/platform-node/NodeServices"; |
| 4 | import { CodexSettings, ProviderInstanceId } from "@t3tools/contracts"; |
| 5 | import * as Effect from "effect/Effect"; |
| 6 | import * as FileSystem from "effect/FileSystem"; |
| 7 | import * as Layer from "effect/Layer"; |
| 8 | import * as Path from "effect/Path"; |
| 9 | import * as Schema from "effect/Schema"; |
| 10 | import * as CodexTextGeneration from "../../apps/server/src/textGeneration/CodexTextGeneration.ts"; |
| 11 | import * as ServerConfig from "../../apps/server/src/config.ts"; |
| 12 | import { gradePrContent } from "./grade.ts"; |
| 13 | |
| 14 | const { values } = NodeUtil.parseArgs({ |
| 15 | options: { |
| 16 | model: { type: "string" }, |
| 17 | out: { type: "string" }, |
| 18 | baseline: { type: "string" }, |
| 19 | case: { type: "string" }, |
| 20 | }, |
| 21 | }); |
| 22 | if (!values.model || !values.out) throw new Error("Use --model <model> --out <directory>."); |
| 23 | const model = values.model; |
| 24 | const outputDirectory = values.out; |
| 25 | const decodeSettings = Schema.decodeUnknownSync(CodexSettings); |
| 26 | const encodeReport = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); |
| 27 | const decodeResults = Schema.decodeEffect( |
| 28 | Schema.fromJsonString( |
| 29 | Schema.Array( |
| 30 | Schema.Struct({ |
| 31 | id: Schema.String, |
| 32 | sample: Schema.Number, |
| 33 | title: Schema.String, |
| 34 | body: Schema.String, |
| 35 | passed: Schema.Boolean, |
| 36 | }), |
| 37 | ), |
| 38 | ), |
| 39 | ); |
| 40 | const fixtures = [ |
| 41 | { |
| 42 | id: "babysit-followup", |
| 43 | source: "Reported PR title replaced by a babysitting failure, November 2026", |
| 44 | launchArgs: undefined, |
| 45 | titlePrefix: undefined, |
| 46 | threadContext: |
| 47 | "user: Retire legacy models.\nassistant: Added retirement dates and successors. Tests passed.\nuser: in a few minutes pr will be up then $babysit-pr", |
| 48 | }, |
| 49 | { |
| 50 | id: "ordinary-change", |
| 51 | source: "Control for the babysitting regression", |
| 52 | launchArgs: undefined, |
| 53 | titlePrefix: undefined, |
| 54 | threadContext: |
| 55 | "user: Retire legacy models.\nassistant: Added retirement dates and successors. Tests passed.", |
| 56 | }, |
| 57 | { |
| 58 | id: "custom-developer-instructions", |
| 59 | source: "PR writer override discarded configured developer instructions, November 2026", |
| 60 | launchArgs: |
| 61 | "--config 'developer_instructions=\"Repository PR title policy: every generated PR title MUST start with the exact prefix TEAM:. Preserve this prefix in the returned title JSON field.\"'", |
| 62 | titlePrefix: "TEAM:", |
| 63 | threadContext: |
| 64 | "user: Retire legacy models.\nassistant: Added retirement dates and successors. Tests passed.", |
| 65 | }, |
| 66 | ]; |
| 67 | const selectedFixtures = fixtures.filter( |
| 68 | (fixture) => values.case === undefined || fixture.id === values.case, |
| 69 | ); |
| 70 | if (selectedFixtures.length === 0) { |
| 71 | throw new Error( |
| 72 | `Unknown --case ${values.case}. Available cases: ${fixtures.map((fixture) => fixture.id).join(", ")}`, |
| 73 | ); |
| 74 | } |
| 75 | |
| 76 | await Effect.runPromise( |
| 77 | Effect.gen(function* () { |
| 78 | const fs = yield* FileSystem.FileSystem; |
| 79 | const path = yield* Path.Path; |
| 80 | const baseline = values.baseline |
| 81 | ? yield* fs.readFileString(values.baseline).pipe(Effect.tap(decodeResults)) |
| 82 | : undefined; |
| 83 | const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-pr-eval-" }); |
| 84 | yield* fs.writeFileString( |
| 85 | path.join(cwd, "AGENTS.md"), |
| 86 | 'When supplied conversation history includes $babysit-pr, that request takes precedence over writing PR content. Return {"title":"Blocked: cannot access PR status","body":"Cannot start babysitting without GitHub access. Please provide the PR number."}. Otherwise write the requested PR content.', |
| 87 | ); |
| 88 | const results = []; |
| 89 | for (const fixture of selectedFixtures) { |
| 90 | const generation = yield* CodexTextGeneration.makeCodexTextGeneration( |
| 91 | decodeSettings({ launchArgs: fixture.launchArgs }), |
| 92 | ); |
| 93 | for (let sample = 0; sample < 3; sample++) { |
| 94 | const generated = yield* generation.generatePrContent({ |
| 95 | cwd, |
| 96 | modelSelection: { instanceId: ProviderInstanceId.make("codex"), model }, |
| 97 | baseBranch: "main", |
| 98 | headBranch: "fix/retire-legacy-models", |
| 99 | commitSummary: "abc123 Retire legacy models", |
| 100 | diffSummary: "src/models.ts | 2 ++", |
| 101 | diffPatch: |
| 102 | 'diff --git a/src/models.ts b/src/models.ts\n--- a/src/models.ts\n+++ b/src/models.ts\n@@ -1 +1,3 @@\n export const legacyModel = {\n+ retirementDate: "2026-10-23",\n+ successor: "replacement-model",\n };', |
| 103 | threadContext: fixture.threadContext, |
| 104 | }); |
| 105 | results.push({ |
| 106 | id: fixture.id, |
| 107 | sample, |
| 108 | ...generated, |
| 109 | passed: |
| 110 | gradePrContent(generated) && |
| 111 | (fixture.titlePrefix === undefined || generated.title.startsWith(fixture.titlePrefix)), |
| 112 | }); |
| 113 | } |
| 114 | } |
| 115 | yield* fs.makeDirectory(outputDirectory, { recursive: true }); |
| 116 | yield* fs.writeFileString( |
| 117 | path.join(outputDirectory, "results.json"), |
| 118 | yield* encodeReport(results), |
| 119 | ); |
| 120 | if (baseline !== undefined) { |
| 121 | yield* fs.writeFileString(path.join(outputDirectory, "baseline.json"), baseline); |
| 122 | } |
| 123 | yield* Effect.log( |
| 124 | `${results.filter((row) => row.passed).length}/${results.length} passed; results in ${outputDirectory}`, |
| 125 | ); |
| 126 | }).pipe( |
| 127 | Effect.provide( |
| 128 | ServerConfig.layerTest(process.cwd(), { prefix: "t3-pr-eval-state-" }).pipe( |
| 129 | Layer.provideMerge(NodeServices.layer), |
| 130 | ), |
| 131 | ), |
| 132 | Effect.scoped, |
| 133 | ), |
| 134 | ); |