The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 03:37 UTC
| 1 | #!/usr/bin/env node |
| 2 | // An eval script, not server code: it shells out to git and reads files directly. |
| 3 | // @effect-diagnostics nodeBuiltinImport:off |
| 4 | import * as NodeChildProcess from "node:child_process"; |
| 5 | import * as NodeCrypto from "node:crypto"; |
| 6 | import * as NodeFS from "node:fs"; |
| 7 | import * as NodeOS from "node:os"; |
| 8 | import * as NodePath from "node:path"; |
| 9 | import * as NodeUtil from "node:util"; |
| 10 | import * as Effect from "effect/Effect"; |
| 11 | import * as Schedule from "effect/Schedule"; |
| 12 | import * as Schema from "effect/Schema"; |
| 13 | import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/http"; |
| 14 | import { proxyConfig } from "../shared/proxy.ts"; |
| 15 | import { cases, type Case } from "./cases.ts"; |
| 16 | import { grade, testRange, type Comment } from "./grade.ts"; |
| 17 | |
| 18 | const { values } = NodeUtil.parseArgs({ |
| 19 | options: { |
| 20 | out: { type: "string" }, |
| 21 | model: { type: "string", multiple: true }, |
| 22 | prompt: { type: "string" }, |
| 23 | case: { type: "string", multiple: true }, |
| 24 | samples: { type: "string", default: "2" }, |
| 25 | concurrency: { type: "string", default: "6" }, |
| 26 | reasoning: { type: "string", default: "low" }, |
| 27 | }, |
| 28 | }); |
| 29 | if (!values.out) throw new Error("Use --out <directory> [--model <id>] [--prompt <file>]."); |
| 30 | const outDirectory = values.out; |
| 31 | const models = values.model ?? ["gpt-6.1-terra", "gpt-6-luna"]; |
| 32 | const repo = NodePath.resolve(import.meta.dirname, "../.."); |
| 33 | const checkFile = |
| 34 | values.prompt ?? NodePath.join(repo, ".macroscope/check-run-agents/test-signal.md"); |
| 35 | // Macroscope gives the model the body; the front matter only configures the run. |
| 36 | const instructions = NodeFS.readFileSync(checkFile, "utf8") |
| 37 | .replace(/^---\n[\s\S]*?\n---\n/, "") |
| 38 | .trim(); |
| 39 | const words = instructions.split(/\s+/).length; |
| 40 | const promptHash = NodeCrypto.createHash("sha256").update(instructions).digest("hex").slice(0, 12); |
| 41 | const selected = values.case ? cases.filter((entry) => values.case!.includes(entry.id)) : cases; |
| 42 | const proxy = proxyConfig(); |
| 43 | |
| 44 | const git = (args: ReadonlyArray<string>) => { |
| 45 | const result = NodeChildProcess.spawnSync("git", args, { |
| 46 | cwd: repo, |
| 47 | encoding: "utf8", |
| 48 | maxBuffer: 1 << 26, |
| 49 | }); |
| 50 | return result.status === 0 ? result.stdout : undefined; |
| 51 | }; |
| 52 | |
| 53 | /** The case's PR head as files the model can read, plus the diff it reviews. */ |
| 54 | const world = (entry: Case) => { |
| 55 | const files = new Map<string, string>(); |
| 56 | if (entry.type === "real") { |
| 57 | const diff = git(["diff", `${entry.rev}~1`, entry.rev, "--", ...entry.paths]); |
| 58 | // An empty diff would grade a guard case as a clean pass. |
| 59 | if (!diff) throw new Error(`${entry.id}: no diff at ${entry.rev}.`); |
| 60 | return { diff, files }; |
| 61 | } |
| 62 | const before = git(["show", `${entry.rev}:${entry.path}`]); |
| 63 | if (!before?.includes(entry.after)) throw new Error(`${entry.id}: anchor not found.`); |
| 64 | const after = before.replace(entry.after, entry.after + entry.insert); |
| 65 | files.set(entry.path, after); |
| 66 | const directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-test-signal-")); |
| 67 | NodeFS.writeFileSync(NodePath.join(directory, "before"), before); |
| 68 | NodeFS.writeFileSync(NodePath.join(directory, "after"), after); |
| 69 | const raw = NodeChildProcess.spawnSync("git", ["diff", "--no-index", "before", "after"], { |
| 70 | cwd: directory, |
| 71 | encoding: "utf8", |
| 72 | }).stdout; |
| 73 | NodeFS.rmSync(directory, { recursive: true }); |
| 74 | const diff = raw |
| 75 | .replace("diff --git a/before b/after", `diff --git a/${entry.path} b/${entry.path}`) |
| 76 | .replace("--- a/before", `--- a/${entry.path}`) |
| 77 | .replace("+++ b/after", `+++ b/${entry.path}`); |
| 78 | return { diff, files }; |
| 79 | }; |
| 80 | |
| 81 | // The check reads the rules at the PR head, where they always exist. Old case commits predate |
| 82 | // them, so serve the rules from this checkout. |
| 83 | const ruleFiles = new Set(["docs/internals/testing.md", "AGENTS.md"]); |
| 84 | const readAt = (entry: Case, files: Map<string, string>, path: string) => |
| 85 | ruleFiles.has(path) |
| 86 | ? NodeFS.readFileSync(NodePath.join(repo, path), "utf8") |
| 87 | : (files.get(path) ?? git(["show", `${entry.rev}:${path}`])); |
| 88 | |
| 89 | /** `git grep` at `rev`, with overlay and rule files searched in their served form instead. */ |
| 90 | const grepAt = (entry: Case, files: Map<string, string>, pattern: string, scope: string) => { |
| 91 | // Status 2 is a bad pattern. Show grep's message, as a real tool would, so the model can fix it. |
| 92 | const check = NodeChildProcess.spawnSync("grep", ["-E", "--", pattern], { |
| 93 | input: "", |
| 94 | encoding: "utf8", |
| 95 | }); |
| 96 | if (check.status !== 0 && check.status !== 1) return check.stderr.trim() || "grep failed."; |
| 97 | const served = new Map(files); |
| 98 | for (const path of ruleFiles) served.set(path, readAt(entry, files, path) ?? ""); |
| 99 | // Served files also exist in this checkout, so git applies its own pathspec rules to the scope. |
| 100 | const matched = new Set((git(["ls-files", "--", scope || "."]) ?? "").split("\n")); |
| 101 | const inScope = (path: string) => matched.has(path); |
| 102 | const fromRev = (git(["grep", "-n", "-E", "-e", pattern, entry.rev, "--", scope || "."]) ?? "") |
| 103 | .split("\n") |
| 104 | .filter((line) => line !== "") |
| 105 | .map((line) => line.replace(`${entry.rev}:`, "")) |
| 106 | .filter((line) => !served.has(line.slice(0, line.indexOf(":")))); |
| 107 | const fromServed = [...served] |
| 108 | .filter(([path]) => inScope(path)) |
| 109 | .flatMap(([path, text]) => |
| 110 | NodeChildProcess.spawnSync("grep", ["-n", "-E", "--", pattern], { |
| 111 | input: text, |
| 112 | encoding: "utf8", |
| 113 | }) |
| 114 | .stdout.split("\n") |
| 115 | .filter((line) => line !== "") |
| 116 | .map((line) => `${path}:${line}`), |
| 117 | ); |
| 118 | return [...fromServed, ...fromRev].slice(0, 80).join("\n") || "No matches."; |
| 119 | }; |
| 120 | |
| 121 | const tools = [ |
| 122 | { |
| 123 | type: "function", |
| 124 | name: "read_file", |
| 125 | description: "Read a file at the pull request head, with line numbers.", |
| 126 | parameters: { |
| 127 | type: "object", |
| 128 | properties: { |
| 129 | path: { type: "string" }, |
| 130 | start_line: { type: "integer" }, |
| 131 | end_line: { type: "integer" }, |
| 132 | }, |
| 133 | required: ["path"], |
| 134 | }, |
| 135 | }, |
| 136 | { |
| 137 | type: "function", |
| 138 | name: "grep", |
| 139 | description: |
| 140 | "Search the repository at the pull request head with an extended regular expression.", |
| 141 | parameters: { |
| 142 | type: "object", |
| 143 | properties: { pattern: { type: "string" }, path: { type: "string" } }, |
| 144 | required: ["pattern"], |
| 145 | }, |
| 146 | }, |
| 147 | { |
| 148 | type: "function", |
| 149 | name: "post_comment", |
| 150 | description: "Post an inline review comment on a line of the pull request head.", |
| 151 | parameters: { |
| 152 | type: "object", |
| 153 | properties: { path: { type: "string" }, line: { type: "integer" }, body: { type: "string" } }, |
| 154 | required: ["path", "line", "body"], |
| 155 | }, |
| 156 | }, |
| 157 | ]; |
| 158 | |
| 159 | const system = (body: string) => |
| 160 | [ |
| 161 | "You are a Macroscope Check Run Agent reviewing a GitHub pull request.", |
| 162 | "Follow the check instructions below. Read the repository at the pull request head with read_file and grep.", |
| 163 | "Leave inline review comments with post_comment. Then give your final response.", |
| 164 | "", |
| 165 | "<check_instructions>", |
| 166 | body, |
| 167 | "</check_instructions>", |
| 168 | ].join("\n"); |
| 169 | |
| 170 | const decodeResponse = Schema.decodeUnknownSync( |
| 171 | Schema.fromJsonString( |
| 172 | Schema.Struct({ |
| 173 | output: Schema.Array(Schema.Record(Schema.String, Schema.Unknown)), |
| 174 | usage: Schema.optional(Schema.Struct({ total_tokens: Schema.optional(Schema.Number) })), |
| 175 | }), |
| 176 | ), |
| 177 | ); |
| 178 | const decodeArguments = Schema.decodeUnknownSync( |
| 179 | Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)), |
| 180 | ); |
| 181 | const encodeRow = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)); |
| 182 | |
| 183 | const post = Effect.fn("evaluateTestSignal.post")(function* (body: unknown) { |
| 184 | const client = (yield* HttpClient.HttpClient).pipe( |
| 185 | HttpClient.retryTransient({ |
| 186 | retryOn: "errors-and-responses", |
| 187 | schedule: Schedule.exponential("2 seconds"), |
| 188 | times: 4, |
| 189 | }), |
| 190 | ); |
| 191 | const response = yield* HttpClientRequest.post(`${proxy.url}/responses`).pipe( |
| 192 | HttpClientRequest.setHeaders(proxy.headers), |
| 193 | HttpClientRequest.bodyJson(body), |
| 194 | Effect.flatMap(client.execute), |
| 195 | ); |
| 196 | const text = yield* response.text; |
| 197 | if (response.status !== 200) |
| 198 | return yield* Effect.die(new Error(`Model API ${response.status}: ${text}`)); |
| 199 | return decodeResponse(text); |
| 200 | }); |
| 201 | |
| 202 | const runSample = Effect.fn("evaluateTestSignal.sample")(function* (entry: Case, model: string) { |
| 203 | const { diff, files } = world(entry); |
| 204 | const comments: Array<Comment> = []; |
| 205 | const input: Array<unknown> = [ |
| 206 | { |
| 207 | role: "user", |
| 208 | content: `Pull request diff for the files this check includes:\n\n\`\`\`diff\n${diff}\`\`\``, |
| 209 | }, |
| 210 | ]; |
| 211 | let final = ""; |
| 212 | let tokens = 0; |
| 213 | for (let step = 0; step < 16; step++) { |
| 214 | const response = yield* post({ |
| 215 | model, |
| 216 | instructions: system(instructions), |
| 217 | input, |
| 218 | tools, |
| 219 | reasoning: { effort: values.reasoning }, |
| 220 | store: false, |
| 221 | include: ["reasoning.encrypted_content"], |
| 222 | }); |
| 223 | tokens += response.usage?.total_tokens ?? 0; |
| 224 | input.push(...response.output); |
| 225 | const calls = response.output.filter((item) => item.type === "function_call"); |
| 226 | final = response.output |
| 227 | .filter((item) => item.type === "message") |
| 228 | .flatMap((item) => (item.content as Array<{ text?: string }>).map((part) => part.text ?? "")) |
| 229 | .join(""); |
| 230 | if (calls.length === 0) break; |
| 231 | for (const call of calls) { |
| 232 | const args = decodeArguments(String(call.arguments)); |
| 233 | let output: string; |
| 234 | if (call.name === "post_comment") { |
| 235 | comments.push({ |
| 236 | path: String(args.path), |
| 237 | line: Number(args.line), |
| 238 | body: String(args.body), |
| 239 | }); |
| 240 | output = "Posted."; |
| 241 | } else if (call.name === "read_file") { |
| 242 | const text = readAt(entry, files, String(args.path)); |
| 243 | const lines = text?.split("\n") ?? []; |
| 244 | const start = Math.max(1, Number(args.start_line ?? 1)); |
| 245 | const end = Math.min(lines.length, Number(args.end_line ?? start + 399)); |
| 246 | output = |
| 247 | text === undefined |
| 248 | ? "File not found." |
| 249 | : lines |
| 250 | .slice(start - 1, end) |
| 251 | .map((line, index) => `${start + index}: ${line}`) |
| 252 | .join("\n"); |
| 253 | } else { |
| 254 | output = grepAt(entry, files, String(args.pattern), String(args.path ?? "")); |
| 255 | } |
| 256 | input.push({ type: "function_call_output", call_id: call.call_id, output }); |
| 257 | } |
| 258 | } |
| 259 | const targets = entry.targets.flatMap((target) => { |
| 260 | const range = testRange(readAt(entry, files, target.path) ?? "", target.title); |
| 261 | if (!range) throw new Error(`${entry.id}: target "${target.title}" not found.`); |
| 262 | return [{ path: target.path, ...range }]; |
| 263 | }); |
| 264 | return { ...grade({ kind: entry.kind, targets, comments, final }), comments, final, tokens }; |
| 265 | }); |
| 266 | |
| 267 | const sampleSafely = (entry: Case, model: string, sample: number) => |
| 268 | runSample(entry, model).pipe( |
| 269 | Effect.map((result) => ({ case: entry.id, kind: entry.kind, model, sample, ...result })), |
| 270 | Effect.catchCause((cause) => |
| 271 | Effect.succeed({ case: entry.id, kind: entry.kind, model, sample, error: String(cause) }), |
| 272 | ), |
| 273 | ); |
| 274 | |
| 275 | type Row = Effect.Success<ReturnType<typeof sampleSafely>>; |
| 276 | const isGraded = (row: Row) => "pass" in row; |
| 277 | const passed = (row: Row) => "pass" in row && row.pass; |
| 278 | |
| 279 | const summary = await Effect.runPromise( |
| 280 | Effect.gen(function* () { |
| 281 | NodeFS.mkdirSync(outDirectory, { recursive: true }); |
| 282 | const samples = Number(values.samples); |
| 283 | const pairs = models.flatMap((model) => selected.map((entry) => ({ model, entry }))); |
| 284 | const rows: Array<Row> = (yield* Effect.forEach( |
| 285 | pairs.flatMap(({ model, entry }) => |
| 286 | Array.from({ length: samples }, (_, n) => ({ model, entry, n })), |
| 287 | ), |
| 288 | ({ model, entry, n }) => sampleSafely(entry, model, n), |
| 289 | { concurrency: Number(values.concurrency) }, |
| 290 | )).slice(); |
| 291 | // A third sample settles a split. |
| 292 | const splits = pairs.filter(({ model, entry }) => { |
| 293 | const own = rows.filter((row) => row.model === model && row.case === entry.id); |
| 294 | return own.length === 2 && own.some(passed) && !own.every(passed); |
| 295 | }); |
| 296 | rows.push( |
| 297 | ...(yield* Effect.forEach(splits, ({ model, entry }) => sampleSafely(entry, model, samples), { |
| 298 | concurrency: Number(values.concurrency), |
| 299 | })), |
| 300 | ); |
| 301 | NodeFS.writeFileSync( |
| 302 | NodePath.join(outDirectory, "results.ndjson"), |
| 303 | rows.map((row) => encodeRow(row)).join("\n") + "\n", |
| 304 | ); |
| 305 | |
| 306 | const lines = [`prompt ${promptHash}, ${words} words, ${checkFile}`, ""]; |
| 307 | lines.push("| model | bad caught | guards clean | errors | tokens |", "|---|---|---|---|---|"); |
| 308 | for (const model of models) { |
| 309 | const verdicts = selected.map((entry) => { |
| 310 | const own = rows.filter((row) => row.model === model && row.case === entry.id); |
| 311 | // A run error counts as a failed sample, so one pass and one error is not a majority. |
| 312 | const graded = own.filter(isGraded); |
| 313 | const passes = own.filter(passed).length; |
| 314 | return { |
| 315 | entry, |
| 316 | pass: passes > own.length / 2, |
| 317 | errors: own.length - graded.length, |
| 318 | }; |
| 319 | }); |
| 320 | const count = (kind: Case["kind"]) => { |
| 321 | const own = verdicts.filter((verdict) => verdict.entry.kind === kind); |
| 322 | return `${own.filter((verdict) => verdict.pass).length}/${own.length}`; |
| 323 | }; |
| 324 | const tokens = rows |
| 325 | .filter((row) => row.model === model && "tokens" in row) |
| 326 | .reduce((sum, row) => sum + ("tokens" in row ? row.tokens : 0), 0); |
| 327 | const errors = verdicts.reduce((sum, verdict) => sum + verdict.errors, 0); |
| 328 | lines.push(`| ${model} | ${count("bad")} | ${count("guard")} | ${errors} | ${tokens} |`); |
| 329 | const failed = verdicts.filter((verdict) => !verdict.pass).map((verdict) => verdict.entry.id); |
| 330 | if (failed.length > 0) lines.push(`| failed: ${failed.join(", ")} | | | | |`); |
| 331 | } |
| 332 | const report = lines.join("\n") + "\n"; |
| 333 | NodeFS.writeFileSync(NodePath.join(outDirectory, "summary.md"), report); |
| 334 | return report; |
| 335 | }).pipe(Effect.provide(FetchHttpClient.layer)), |
| 336 | ); |
| 337 | process.stdout.write(summary); |