@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 364 lines 12.8 KB
Raw
1 import {
2 NodeId,
3 ProviderInstanceId,
4 ProviderThreadId,
5 ProviderTurnId,
6 RunId,
7 type OrchestrationV2ProviderTurn,
8 type ServerProvider,
9 type TurnTokenUsage,
10 } from "@t3tools/contracts";
11 import { describe, expect, it } from "@effect/vitest";
12 import * as DateTime from "effect/DateTime";
13 import * as Effect from "effect/Effect";
14 import * as Layer from "effect/Layer";
15
16 import * as ProviderRegistry from "../provider/ProviderRegistry.ts";
17 import * as AnalyticsService from "./AnalyticsService.ts";
18 import { catalogModelLabel } from "./ProviderDimensions.ts";
19 import {
20 ProviderTurnAnalytics,
21 RequestUsageLedger,
22 layer,
23 providerTurnProperties,
24 type ProviderTurnAnalyticsContext,
25 type ProviderTurnSegment,
26 } from "./ProviderUsageAnalytics.ts";
27
28 const providers = [
29 // Codex lists whatever its configured model provider serves, including private deployments.
30 { driver: "codex", models: [{ slug: "gpt-6-sol" }, { slug: "acme-azure-deployment" }] },
31 { driver: "claudeAgent", models: [{ slug: "claude-opus-5-5" }, { slug: "claude-haiku-4-5" }] },
32 { driver: "opencode", models: [{ slug: "ollama/acme-internal" }] },
33 // Antigravity ships models before the bundled manifest lists them.
34 {
35 driver: "antigravity",
36 models: [
37 { slug: "gemini-4-neon", aliases: ["antigravity-default"] },
38 { slug: "gemini-4-hydrogen" },
39 ],
40 },
41 ].map(
42 ({ driver, models }) =>
43 ({
44 instanceId: ProviderInstanceId.make(driver),
45 driver,
46 models: models.map((model) => ({ ...model, isCustom: false })),
47 }) as unknown as ServerProvider,
48 );
49 const label = (model: string) => catalogModelLabel(providers, model);
50
51 const at = (ms: number) => DateTime.makeUnsafe(ms);
52 const complete = (input: number, output: number, extra: Partial<TurnTokenUsage> = {}) =>
53 ({
54 usageStatus: "complete",
55 usageScope: "main_agent",
56 hasSubagents: false,
57 inputTokens: input,
58 outputTokens: output,
59 ...extra,
60 }) as TurnTokenUsage;
61
62 const turn = (
63 id: string,
64 usage: TurnTokenUsage | undefined,
65 status: OrchestrationV2ProviderTurn["status"] = "completed",
66 ): OrchestrationV2ProviderTurn => ({
67 id: ProviderTurnId.make(id),
68 providerThreadId: ProviderThreadId.make("provider-thread"),
69 nodeId: NodeId.make(`node:${id}`),
70 runAttemptId: null,
71 nativeTurnRef: null,
72 ordinal: 1,
73 status,
74 startedAt: at(1_000),
75 completedAt: at(4_000),
76 ...(usage === undefined ? {} : { turnTokenUsage: usage }),
77 });
78
79 const context = (
80 runId: string,
81 segment: ProviderTurnSegment,
82 model = "gpt-6-sol",
83 ): ProviderTurnAnalyticsContext => ({
84 runId: RunId.make(runId),
85 modelSelection: {
86 instanceId: ProviderInstanceId.make("codex"),
87 model,
88 options: [
89 { id: "reasoningEffort", value: "xhigh" },
90 { id: "serviceTier", value: "priority" },
91 ],
92 },
93 runtimeMode: "full-access",
94 interactionMode: "default",
95 origin: { createdBy: "user", creationSource: "web", scheduled: false, subagent: false },
96 segment,
97 });
98
99 const settled = (runId: string, childrenRunning = false) => ({
100 runId: RunId.make(runId),
101 status: "completed" as const,
102 completedAt: at(9_000),
103 childrenRunning,
104 });
105
106 describe("provider usage reporting", () => {
107 it.effect(
108 "records one request across a continuation, a child, a replay, and a late child",
109 () => {
110 const recorded: Array<readonly [string, unknown]> = [];
111 const layerTest = layer.pipe(
112 Layer.provide(
113 Layer.mergeAll(
114 Layer.mock(AnalyticsService.AnalyticsService)({
115 record: (event, properties) =>
116 Effect.sync(() => void recorded.push([event, properties])),
117 }),
118 Layer.mock(ProviderRegistry.ProviderRegistry)({
119 getProviders: Effect.succeed(providers),
120 }),
121 ),
122 ),
123 );
124 return Effect.gen(function* () {
125 const analytics = yield* ProviderTurnAnalytics;
126 const root = context("run-a", "initial");
127 yield* analytics.runStarted({ runId: RunId.make("run-a"), driver: "codex", context: root });
128 const first = { driver: "codex", context: root };
129 const firstTurn = turn("root", complete(100, 10, { models: ["gpt-6-sol"] }));
130 yield* analytics.turnCompleted({ ...first, providerTurn: firstTurn });
131 yield* analytics.turnCompleted({ ...first, providerTurn: firstTurn });
132 yield* analytics.turnCompleted({
133 driver: "codex",
134 context: context("run-a", "continuation"),
135 providerTurn: turn("goal", complete(50, 5, { models: ["gpt-6-sol"] })),
136 });
137 yield* analytics.turnCompleted({
138 driver: "codex",
139 context: context("run-a", "native_child"),
140 providerTurn: turn("child", complete(20, 2)),
141 });
142 yield* analytics.runSettled(settled("run-a"));
143 yield* analytics.turnCompleted({
144 driver: "codex",
145 context: context("run-a", "native_child"),
146 providerTurn: turn("late-child", complete(7, 1)),
147 });
148 yield* analytics.runSettled(settled("run-a"));
149 yield* analytics.runStarted({
150 runId: RunId.make("run-b"),
151 driver: "codex",
152 context: context("run-b", "initial"),
153 });
154 yield* analytics.runSettled(settled("run-b"));
155
156 const requests = recorded.filter(([event]) => event === "provider.request.completed");
157 expect(recorded.filter(([event]) => event === "provider.turn.completed")).toHaveLength(5);
158 expect(requests).toHaveLength(2);
159 // The whole payload: no run, turn, or thread ids, prompts, or raw model names.
160 expect(requests[0]?.[1]).toEqual({
161 reportVersion: 2,
162 provider: "codex",
163 requestKind: "user",
164 terminalStatus: "completed",
165 lifecycleObserved: true,
166 model: "gpt-6-sol",
167 effort: "xhigh",
168 serviceTier: "fast",
169 runtimeMode: "full-access",
170 interactionMode: "default",
171 createdBy: "user",
172 creationSource: "web",
173 scheduled: false,
174 subagent: false,
175 providerTurns: 2,
176 continuations: 1,
177 steeringRestarts: 0,
178 usageStatus: "complete",
179 inputTokens: 150,
180 outputTokens: 15,
181 nativeChildTurns: 1,
182 childUsageStatus: "complete",
183 childInputTokens: 20,
184 childOutputTokens: 2,
185 durationMs: 8_000,
186 });
187 expect(requests[1]?.[1]).toMatchObject({ providerTurns: 0, usageStatus: "unavailable" });
188 }).pipe(Effect.provide(layerTest));
189 },
190 );
191
192 it("keeps a request partial when its start, a steered segment, or a counter is missing", () => {
193 const ledger = new RequestUsageLedger();
194 ledger.start(RunId.make("restarted"), "codex", {
195 ...context("restarted", "initial"),
196 restartContinuation: true,
197 });
198 ledger.add({
199 driver: "codex",
200 context: context("restarted", "initial"),
201 providerTurn: turn("after-restart", complete(10, 1)),
202 });
203 expect(ledger.settle(settled("restarted"), label)).toMatchObject({
204 restartContinuation: true,
205 lifecycleObserved: false,
206 usageStatus: "partial",
207 inputTokens: 10,
208 });
209
210 ledger.start(RunId.make("steered"), "claudeAgent", context("steered", "initial"));
211 ledger.add({
212 driver: "claudeAgent",
213 context: context("steered", "initial"),
214 providerTurn: turn(
215 "steered-away",
216 { usageStatus: "partial", usageScope: "main_agent", hasSubagents: false, outputTokens: 4 },
217 "interrupted",
218 ),
219 });
220 ledger.add({
221 driver: "claudeAgent",
222 context: context("steered", "steering_restart"),
223 providerTurn: turn("steered-to", complete(30, 6)),
224 });
225 expect(ledger.settle(settled("steered"), label)).toMatchObject({
226 usageStatus: "partial",
227 steeringRestarts: 1,
228 inputTokens: 30,
229 outputTokens: 10,
230 });
231
232 ledger.start(RunId.make("silent"), "grok", context("silent", "initial"));
233 ledger.add({
234 driver: "grok",
235 context: context("silent", "initial"),
236 providerTurn: turn("silent", undefined),
237 });
238 const silent = ledger.settle(settled("silent", true), label);
239 expect(silent).toMatchObject({
240 usageStatus: "unavailable",
241 childrenRunning: true,
242 childUsageStatus: "unavailable",
243 });
244 expect(silent).not.toHaveProperty("outputTokens");
245 });
246
247 it("confirms only the models a provider reported", () => {
248 const opus = context("models", "initial", "claude-opus-5-5");
249 const properties = (usage: TurnTokenUsage | undefined) =>
250 providerTurnProperties(
251 { driver: "claudeAgent", context: opus, providerTurn: turn("t", usage) },
252 label,
253 );
254 expect(properties(complete(1, 1, { models: ["claude-haiku-4-5-20251001"] }))).toMatchObject({
255 model: "claude-opus-5-5",
256 observedModel: "claude-haiku-4-5",
257 mixedModels: false,
258 });
259 const unconfirmed = properties(complete(1, 1));
260 expect(unconfirmed).toMatchObject({ model: "claude-opus-5-5" });
261 expect(unconfirmed).not.toHaveProperty("mixedModels");
262 expect(properties(complete(1, 1, { models: ["a", "b"] }))).toMatchObject({ mixedModels: true });
263
264 const ledger = new RequestUsageLedger();
265 ledger.start(RunId.make("mixed"), "codex", context("mixed", "initial"));
266 ledger.add({
267 driver: "codex",
268 context: context("mixed", "initial"),
269 providerTurn: turn("root", complete(1, 1, { models: ["gpt-6-sol"] })),
270 });
271 ledger.add({
272 driver: "codex",
273 context: context("mixed", "native_child"),
274 providerTurn: turn("child", complete(1, 1, { models: ["gpt-6-luna"] })),
275 });
276 expect(ledger.settle(settled("mixed"), label)).toMatchObject({ mixedModels: true });
277 });
278
279 it("keeps zero, absent, local-command, and ledger usage distinct", () => {
280 const properties = (usage: TurnTokenUsage | undefined) =>
281 providerTurnProperties(
282 { driver: "claudeAgent", context: context("z", "initial"), providerTurn: turn("t", usage) },
283 label,
284 );
285 expect(properties(complete(0, 0))).toMatchObject({
286 usageStatus: "complete",
287 usageReported: true,
288 inputTokens: 0,
289 outputTokens: 0,
290 });
291 const absent = properties(undefined);
292 expect(absent).toMatchObject({ usageStatus: "unavailable", usageReported: false });
293 expect(absent).not.toHaveProperty("outputTokens");
294 expect(
295 properties(
296 complete(0, 0, { localCommand: true, otherUsage: { inputTokens: 900, outputTokens: 80 } }),
297 ),
298 ).toMatchObject({ localCommand: true, otherInputTokens: 900, otherOutputTokens: 80 });
299 expect(properties(complete(74_167, 236, { reasoningTokens: 30 }))).toMatchObject({
300 outputTokens: 236,
301 reasoningTokens: 30,
302 });
303 });
304
305 it("reports routed catalog models and closed efforts only", () => {
306 const routed = (model: string, variant: string) =>
307 providerTurnProperties(
308 {
309 driver: "opencode",
310 context: {
311 modelSelection: {
312 instanceId: ProviderInstanceId.make("opencode"),
313 model,
314 options: [{ id: "variant", value: variant }],
315 },
316 },
317 providerTurn: turn("t", complete(1, 1, { models: [model] })),
318 },
319 label,
320 );
321 expect(routed("my-proxy/claude-opus-5-5", "high")).toMatchObject({
322 model: "claude-opus-5-5",
323 observedModel: "claude-opus-5-5",
324 effort: "high",
325 });
326 for (const local of [
327 routed("ollama/acme-internal", "secret-variant"),
328 routed("acme-azure-deployment", "high"),
329 ]) {
330 expect(local).not.toHaveProperty("model");
331 expect(local).not.toHaveProperty("observedModel");
332 }
333 expect(routed("ollama/acme-internal", "secret-variant")).not.toHaveProperty("effort");
334 });
335
336 it("reports Antigravity catalog models the bundled manifest does not list", () => {
337 expect(label("gemini-4-hydrogen")).toBe("gemini-4-hydrogen");
338 expect(label("antigravity-default")).toBe("gemini-4-neon");
339 expect(label("gemini-4-unlisted")).toBeUndefined();
340 });
341
342 it("reports closed service tiers from a tier option or a fast mode toggle", () => {
343 const tier = (option: { id: string; value: string | boolean }) =>
344 providerTurnProperties(
345 {
346 driver: "codex",
347 context: {
348 modelSelection: {
349 instanceId: ProviderInstanceId.make("codex"),
350 model: "gpt-6-sol",
351 options: [option],
352 },
353 },
354 providerTurn: turn("t", complete(1, 1)),
355 },
356 label,
357 ).serviceTier;
358 expect(tier({ id: "serviceTier", value: "ultrafast" })).toBe("ultrafast");
359 expect(tier({ id: "serviceTier", value: "default" })).toBe("standard");
360 expect(tier({ id: "serviceTier", value: "/home/me/secret" })).toBeUndefined();
361 expect(tier({ id: "fastMode", value: true })).toBe("fast");
362 expect(tier({ id: "fastMode", value: false })).toBe("standard");
363 });
364 });