@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

main
ts 171 lines 7.15 KB
Raw
1 #!/usr/bin/env node
2 // @effect-diagnostics nodeBuiltinImport:off
3 import * as NodeUtil from "node:util";
4 import * as NodeCrypto from "node:crypto";
5 import * as FetchHttpClient from "effect/http/FetchHttpClient";
6 import * as NodePlatformCrypto from "@effect/platform-node/NodeCrypto";
7 import * as NodeServices from "@effect/platform-node/NodeServices";
8 import { CodexSettings, ProviderInstanceId } from "@t3tools/contracts";
9 import * as Effect from "effect/Effect";
10 import * as Layer from "effect/Layer";
11 import * as Duration from "effect/Duration";
12 import * as FileSystem from "effect/FileSystem";
13 import * as Path from "effect/Path";
14 import * as Schema from "effect/Schema";
15 import * as CodexTextGeneration from "../../apps/server/src/textGeneration/CodexTextGeneration.ts";
16 import { threadTitleEvaluationCases } from "./cases.ts";
17 import {
18 formatThreadTitleContext,
19 type ThreadTitleMessage,
20 } from "@t3tools/provider-core/server/threadTitleContext";
21 import * as ThreadTitleLinks from "../../apps/server/src/textGeneration/ThreadTitleLinks.ts";
22 import * as SourceControlProviderRegistry from "../../apps/server/src/sourceControl/SourceControlProviderRegistry.ts";
23 import * as SourceControlBuiltInDrivers from "../../apps/server/src/sourceControl/builtInDrivers.ts";
24 import * as ServerSettings from "../../apps/server/src/serverSettings.ts";
25 import * as VcsProcess from "../../apps/server/src/vcs/VcsProcess.ts";
26 import * as VcsDriverRegistry from "../../apps/server/src/vcs/VcsDriverRegistry.ts";
27 import * as VcsProjectConfig from "../../apps/server/src/vcs/VcsProjectConfig.ts";
28 import * as GitVcsDriver from "../../apps/server/src/vcs/GitVcsDriver.ts";
29 import * as ProcessRunner from "../../apps/server/src/processRunner.ts";
30 import * as ServerConfig from "../../apps/server/src/config.ts";
31
32 const { values } = NodeUtil.parseArgs({
33 options: {
34 model: { type: "string" },
35 case: { type: "string", multiple: true },
36 out: { type: "string" },
37 baseline: { type: "string" },
38 initial: { type: "boolean", default: false },
39 },
40 });
41 if (!values.model || !values.out)
42 throw new Error("Use --model <configured-model> --out <directory>.");
43 for (const id of values.case ?? []) {
44 if (!threadTitleEvaluationCases.some((fixture) => fixture.id === id)) {
45 throw new Error(`Unknown case id: ${id}.`);
46 }
47 }
48 const model = values.model;
49 const outputDirectory = values.out;
50 const Results = Schema.fromJsonString(
51 Schema.Array(
52 Schema.Struct({
53 id: Schema.String,
54 title: Schema.String,
55 latencyMs: Schema.Number,
56 linkedContextDigest: Schema.String,
57 }),
58 ),
59 );
60 const decodeResults = Schema.decodeUnknownEffect(Results);
61 const decodeSettings = Schema.decodeUnknownEffect(CodexSettings);
62 const encodeReport = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown));
63
64 await Effect.runPromise(
65 Effect.gen(function* () {
66 const fs = yield* FileSystem.FileSystem;
67 const path = yield* Path.Path;
68 const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-title-evaluation-" });
69 const generation = yield* CodexTextGeneration.makeCodexTextGeneration(
70 yield* decodeSettings({}),
71 );
72 const baseline = values.baseline
73 ? yield* fs.readFileString(values.baseline).pipe(Effect.flatMap(decodeResults))
74 : [];
75 const results = [];
76 const review = [];
77 const answerKey = [];
78 for (const fixture of threadTitleEvaluationCases) {
79 if (values.case?.length && !values.case.includes(fixture.id)) continue;
80 const previous = baseline.find((entry) => entry.id === fixture.id);
81 if (values.baseline && !previous) throw new Error(`Baseline is missing ${fixture.id}.`);
82 const firstMessage: ThreadTitleMessage | undefined = fixture.messages.find(
83 (message) => message.role === "user",
84 );
85 if (!firstMessage) throw new Error(`Fixture ${fixture.id} has no user message.`);
86 const context = formatThreadTitleContext(fixture.messages);
87 const message = values.initial ? firstMessage.text : context.message;
88 const attachments = values.initial ? firstMessage.attachments : context.attachments;
89 const [elapsed, { generated, linkedContextDigest }] = yield* Effect.gen(function* () {
90 const linkedContext = yield* ThreadTitleLinks.resolveThreadTitleLinks({
91 cwd,
92 message,
93 });
94 const linkedContextDigest = NodeCrypto.createHash("sha256")
95 .update(linkedContext ?? "")
96 .digest("hex");
97 if (previous && previous.linkedContextDigest !== linkedContextDigest) {
98 throw new Error(
99 `Linked context changed for ${fixture.id}. Record a new baseline before comparing titles.`,
100 );
101 }
102 const generated = yield* generation.generateThreadTitle({
103 cwd,
104 message,
105 ...("projectTitle" in fixture ? { projectTitle: fixture.projectTitle } : {}),
106 ...("instructions" in fixture ? { instructions: fixture.instructions } : {}),
107 previousTitle: values.initial ? undefined : fixture.previousTitle,
108 attachments,
109 linkedContext,
110 modelSelection: { instanceId: ProviderInstanceId.make("codex"), model },
111 });
112 return { generated, linkedContextDigest };
113 }).pipe(Effect.timed);
114 const oldTitle = previous?.title ?? fixture.previousTitle;
115 const newFirst = NodeCrypto.randomInt(2) === 0;
116 results.push({
117 id: fixture.id,
118 title: generated.title,
119 latencyMs: Duration.toMillis(elapsed),
120 needsRefinement: generated.needsRefinement ?? false,
121 linkedContextDigest,
122 });
123 review.push({
124 id: fixture.id,
125 source: fixture.source,
126 request: fixture.request,
127 rubric: fixture.rubric,
128 A: newFirst ? generated.title : oldTitle,
129 B: newFirst ? oldTitle : generated.title,
130 preferred: "",
131 subjectAccuracy: "",
132 recognitionAmongNearbyThreads: "",
133 });
134 answerKey.push({ id: fixture.id, candidate: newFirst ? "A" : "B" });
135 }
136 yield* fs.makeDirectory(outputDirectory, { recursive: true });
137 for (const [name, report] of [
138 ["results", results],
139 ["review", review],
140 ["answer-key", answerKey],
141 ] as const) {
142 yield* fs.writeFileString(
143 path.join(outputDirectory, `${name}.json`),
144 yield* encodeReport(report),
145 );
146 }
147 yield* Effect.log(
148 `Wrote ${results.length} cases to ${outputDirectory}. Score review.json before opening answer-key.json. Latency is in results.json.`,
149 );
150 }).pipe(
151 Effect.provide(
152 Layer.mergeAll(
153 ProcessRunner.layer,
154 SourceControlProviderRegistry.layer.pipe(
155 Layer.provide(SourceControlBuiltInDrivers.layer),
156 Layer.provide(ServerSettings.layerTest()),
157 Layer.provide(VcsDriverRegistry.layer.pipe(Layer.provide(VcsProjectConfig.layer))),
158 Layer.provide(GitVcsDriver.layer),
159 Layer.provide(VcsProcess.layer),
160 Layer.provide(FetchHttpClient.layer),
161 ),
162 ).pipe(
163 Layer.provideMerge(
164 ServerConfig.layerTest(process.cwd(), { prefix: "t3-title-evaluation-state-" }),
165 ),
166 Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodePlatformCrypto.layer)),
167 ),
168 ),
169 Effect.scoped,
170 ),
171 );