The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 04:45 UTC
| 1 | import { |
| 2 | NodeId, |
| 3 | ProviderInstanceId, |
| 4 | ProviderThreadId, |
| 5 | ProviderTurnId, |
| 6 | RunId, |
| 7 | type OrchestrationV2ProviderTurn, |
| 8 | type ServerProvider, |
| 9 | type TurnTokenUsage, |
| 10 | } from "@t3tools/contracts"; |
| 11 | import { describe, expect, it } from "@effect/vitest"; |
| 12 | import * as DateTime from "effect/DateTime"; |
| 13 | import * as Effect from "effect/Effect"; |
| 14 | import * as Layer from "effect/Layer"; |
| 15 | |
| 16 | import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; |
| 17 | import * as AnalyticsService from "./AnalyticsService.ts"; |
| 18 | import { catalogModelLabel } from "./ProviderDimensions.ts"; |
| 19 | import { |
| 20 | ProviderTurnAnalytics, |
| 21 | RequestUsageLedger, |
| 22 | layer, |
| 23 | providerTurnProperties, |
| 24 | type ProviderTurnAnalyticsContext, |
| 25 | type ProviderTurnSegment, |
| 26 | } from "./ProviderUsageAnalytics.ts"; |
| 27 | |
| 28 | const providers = [ |
| 29 | // Codex lists whatever its configured model provider serves, including private deployments. |
| 30 | { driver: "codex", models: [{ slug: "gpt-6-sol" }, { slug: "acme-azure-deployment" }] }, |
| 31 | { driver: "claudeAgent", models: [{ slug: "claude-opus-5-5" }, { slug: "claude-haiku-4-5" }] }, |
| 32 | { driver: "opencode", models: [{ slug: "ollama/acme-internal" }] }, |
| 33 | // Antigravity ships models before the bundled manifest lists them. |
| 34 | { |
| 35 | driver: "antigravity", |
| 36 | models: [ |
| 37 | { slug: "gemini-4-neon", aliases: ["antigravity-default"] }, |
| 38 | { slug: "gemini-4-hydrogen" }, |
| 39 | ], |
| 40 | }, |
| 41 | ].map( |
| 42 | ({ driver, models }) => |
| 43 | ({ |
| 44 | instanceId: ProviderInstanceId.make(driver), |
| 45 | driver, |
| 46 | models: models.map((model) => ({ ...model, isCustom: false })), |
| 47 | }) as unknown as ServerProvider, |
| 48 | ); |
| 49 | const label = (model: string) => catalogModelLabel(providers, model); |
| 50 | |
| 51 | const at = (ms: number) => DateTime.makeUnsafe(ms); |
| 52 | const complete = (input: number, output: number, extra: Partial<TurnTokenUsage> = {}) => |
| 53 | ({ |
| 54 | usageStatus: "complete", |
| 55 | usageScope: "main_agent", |
| 56 | hasSubagents: false, |
| 57 | inputTokens: input, |
| 58 | outputTokens: output, |
| 59 | ...extra, |
| 60 | }) as TurnTokenUsage; |
| 61 | |
| 62 | const turn = ( |
| 63 | id: string, |
| 64 | usage: TurnTokenUsage | undefined, |
| 65 | status: OrchestrationV2ProviderTurn["status"] = "completed", |
| 66 | ): OrchestrationV2ProviderTurn => ({ |
| 67 | id: ProviderTurnId.make(id), |
| 68 | providerThreadId: ProviderThreadId.make("provider-thread"), |
| 69 | nodeId: NodeId.make(`node:${id}`), |
| 70 | runAttemptId: null, |
| 71 | nativeTurnRef: null, |
| 72 | ordinal: 1, |
| 73 | status, |
| 74 | startedAt: at(1_000), |
| 75 | completedAt: at(4_000), |
| 76 | ...(usage === undefined ? {} : { turnTokenUsage: usage }), |
| 77 | }); |
| 78 | |
| 79 | const context = ( |
| 80 | runId: string, |
| 81 | segment: ProviderTurnSegment, |
| 82 | model = "gpt-6-sol", |
| 83 | ): ProviderTurnAnalyticsContext => ({ |
| 84 | runId: RunId.make(runId), |
| 85 | modelSelection: { |
| 86 | instanceId: ProviderInstanceId.make("codex"), |
| 87 | model, |
| 88 | options: [ |
| 89 | { id: "reasoningEffort", value: "xhigh" }, |
| 90 | { id: "serviceTier", value: "priority" }, |
| 91 | ], |
| 92 | }, |
| 93 | runtimeMode: "full-access", |
| 94 | interactionMode: "default", |
| 95 | origin: { createdBy: "user", creationSource: "web", scheduled: false, subagent: false }, |
| 96 | segment, |
| 97 | }); |
| 98 | |
| 99 | const settled = (runId: string, childrenRunning = false) => ({ |
| 100 | runId: RunId.make(runId), |
| 101 | status: "completed" as const, |
| 102 | completedAt: at(9_000), |
| 103 | childrenRunning, |
| 104 | }); |
| 105 | |
| 106 | describe("provider usage reporting", () => { |
| 107 | it.effect( |
| 108 | "records one request across a continuation, a child, a replay, and a late child", |
| 109 | () => { |
| 110 | const recorded: Array<readonly [string, unknown]> = []; |
| 111 | const layerTest = layer.pipe( |
| 112 | Layer.provide( |
| 113 | Layer.mergeAll( |
| 114 | Layer.mock(AnalyticsService.AnalyticsService)({ |
| 115 | record: (event, properties) => |
| 116 | Effect.sync(() => void recorded.push([event, properties])), |
| 117 | }), |
| 118 | Layer.mock(ProviderRegistry.ProviderRegistry)({ |
| 119 | getProviders: Effect.succeed(providers), |
| 120 | }), |
| 121 | ), |
| 122 | ), |
| 123 | ); |
| 124 | return Effect.gen(function* () { |
| 125 | const analytics = yield* ProviderTurnAnalytics; |
| 126 | const root = context("run-a", "initial"); |
| 127 | yield* analytics.runStarted({ runId: RunId.make("run-a"), driver: "codex", context: root }); |
| 128 | const first = { driver: "codex", context: root }; |
| 129 | const firstTurn = turn("root", complete(100, 10, { models: ["gpt-6-sol"] })); |
| 130 | yield* analytics.turnCompleted({ ...first, providerTurn: firstTurn }); |
| 131 | yield* analytics.turnCompleted({ ...first, providerTurn: firstTurn }); |
| 132 | yield* analytics.turnCompleted({ |
| 133 | driver: "codex", |
| 134 | context: context("run-a", "continuation"), |
| 135 | providerTurn: turn("goal", complete(50, 5, { models: ["gpt-6-sol"] })), |
| 136 | }); |
| 137 | yield* analytics.turnCompleted({ |
| 138 | driver: "codex", |
| 139 | context: context("run-a", "native_child"), |
| 140 | providerTurn: turn("child", complete(20, 2)), |
| 141 | }); |
| 142 | yield* analytics.runSettled(settled("run-a")); |
| 143 | yield* analytics.turnCompleted({ |
| 144 | driver: "codex", |
| 145 | context: context("run-a", "native_child"), |
| 146 | providerTurn: turn("late-child", complete(7, 1)), |
| 147 | }); |
| 148 | yield* analytics.runSettled(settled("run-a")); |
| 149 | yield* analytics.runStarted({ |
| 150 | runId: RunId.make("run-b"), |
| 151 | driver: "codex", |
| 152 | context: context("run-b", "initial"), |
| 153 | }); |
| 154 | yield* analytics.runSettled(settled("run-b")); |
| 155 | |
| 156 | const requests = recorded.filter(([event]) => event === "provider.request.completed"); |
| 157 | expect(recorded.filter(([event]) => event === "provider.turn.completed")).toHaveLength(5); |
| 158 | expect(requests).toHaveLength(2); |
| 159 | // The whole payload: no run, turn, or thread ids, prompts, or raw model names. |
| 160 | expect(requests[0]?.[1]).toEqual({ |
| 161 | reportVersion: 2, |
| 162 | provider: "codex", |
| 163 | requestKind: "user", |
| 164 | terminalStatus: "completed", |
| 165 | lifecycleObserved: true, |
| 166 | model: "gpt-6-sol", |
| 167 | effort: "xhigh", |
| 168 | serviceTier: "fast", |
| 169 | runtimeMode: "full-access", |
| 170 | interactionMode: "default", |
| 171 | createdBy: "user", |
| 172 | creationSource: "web", |
| 173 | scheduled: false, |
| 174 | subagent: false, |
| 175 | providerTurns: 2, |
| 176 | continuations: 1, |
| 177 | steeringRestarts: 0, |
| 178 | usageStatus: "complete", |
| 179 | inputTokens: 150, |
| 180 | outputTokens: 15, |
| 181 | nativeChildTurns: 1, |
| 182 | childUsageStatus: "complete", |
| 183 | childInputTokens: 20, |
| 184 | childOutputTokens: 2, |
| 185 | durationMs: 8_000, |
| 186 | }); |
| 187 | expect(requests[1]?.[1]).toMatchObject({ providerTurns: 0, usageStatus: "unavailable" }); |
| 188 | }).pipe(Effect.provide(layerTest)); |
| 189 | }, |
| 190 | ); |
| 191 | |
| 192 | it("keeps a request partial when its start, a steered segment, or a counter is missing", () => { |
| 193 | const ledger = new RequestUsageLedger(); |
| 194 | ledger.start(RunId.make("restarted"), "codex", { |
| 195 | ...context("restarted", "initial"), |
| 196 | restartContinuation: true, |
| 197 | }); |
| 198 | ledger.add({ |
| 199 | driver: "codex", |
| 200 | context: context("restarted", "initial"), |
| 201 | providerTurn: turn("after-restart", complete(10, 1)), |
| 202 | }); |
| 203 | expect(ledger.settle(settled("restarted"), label)).toMatchObject({ |
| 204 | restartContinuation: true, |
| 205 | lifecycleObserved: false, |
| 206 | usageStatus: "partial", |
| 207 | inputTokens: 10, |
| 208 | }); |
| 209 | |
| 210 | ledger.start(RunId.make("steered"), "claudeAgent", context("steered", "initial")); |
| 211 | ledger.add({ |
| 212 | driver: "claudeAgent", |
| 213 | context: context("steered", "initial"), |
| 214 | providerTurn: turn( |
| 215 | "steered-away", |
| 216 | { usageStatus: "partial", usageScope: "main_agent", hasSubagents: false, outputTokens: 4 }, |
| 217 | "interrupted", |
| 218 | ), |
| 219 | }); |
| 220 | ledger.add({ |
| 221 | driver: "claudeAgent", |
| 222 | context: context("steered", "steering_restart"), |
| 223 | providerTurn: turn("steered-to", complete(30, 6)), |
| 224 | }); |
| 225 | expect(ledger.settle(settled("steered"), label)).toMatchObject({ |
| 226 | usageStatus: "partial", |
| 227 | steeringRestarts: 1, |
| 228 | inputTokens: 30, |
| 229 | outputTokens: 10, |
| 230 | }); |
| 231 | |
| 232 | ledger.start(RunId.make("silent"), "grok", context("silent", "initial")); |
| 233 | ledger.add({ |
| 234 | driver: "grok", |
| 235 | context: context("silent", "initial"), |
| 236 | providerTurn: turn("silent", undefined), |
| 237 | }); |
| 238 | const silent = ledger.settle(settled("silent", true), label); |
| 239 | expect(silent).toMatchObject({ |
| 240 | usageStatus: "unavailable", |
| 241 | childrenRunning: true, |
| 242 | childUsageStatus: "unavailable", |
| 243 | }); |
| 244 | expect(silent).not.toHaveProperty("outputTokens"); |
| 245 | }); |
| 246 | |
| 247 | it("confirms only the models a provider reported", () => { |
| 248 | const opus = context("models", "initial", "claude-opus-5-5"); |
| 249 | const properties = (usage: TurnTokenUsage | undefined) => |
| 250 | providerTurnProperties( |
| 251 | { driver: "claudeAgent", context: opus, providerTurn: turn("t", usage) }, |
| 252 | label, |
| 253 | ); |
| 254 | expect(properties(complete(1, 1, { models: ["claude-haiku-4-5-20251001"] }))).toMatchObject({ |
| 255 | model: "claude-opus-5-5", |
| 256 | observedModel: "claude-haiku-4-5", |
| 257 | mixedModels: false, |
| 258 | }); |
| 259 | const unconfirmed = properties(complete(1, 1)); |
| 260 | expect(unconfirmed).toMatchObject({ model: "claude-opus-5-5" }); |
| 261 | expect(unconfirmed).not.toHaveProperty("mixedModels"); |
| 262 | expect(properties(complete(1, 1, { models: ["a", "b"] }))).toMatchObject({ mixedModels: true }); |
| 263 | |
| 264 | const ledger = new RequestUsageLedger(); |
| 265 | ledger.start(RunId.make("mixed"), "codex", context("mixed", "initial")); |
| 266 | ledger.add({ |
| 267 | driver: "codex", |
| 268 | context: context("mixed", "initial"), |
| 269 | providerTurn: turn("root", complete(1, 1, { models: ["gpt-6-sol"] })), |
| 270 | }); |
| 271 | ledger.add({ |
| 272 | driver: "codex", |
| 273 | context: context("mixed", "native_child"), |
| 274 | providerTurn: turn("child", complete(1, 1, { models: ["gpt-6-luna"] })), |
| 275 | }); |
| 276 | expect(ledger.settle(settled("mixed"), label)).toMatchObject({ mixedModels: true }); |
| 277 | }); |
| 278 | |
| 279 | it("keeps zero, absent, local-command, and ledger usage distinct", () => { |
| 280 | const properties = (usage: TurnTokenUsage | undefined) => |
| 281 | providerTurnProperties( |
| 282 | { driver: "claudeAgent", context: context("z", "initial"), providerTurn: turn("t", usage) }, |
| 283 | label, |
| 284 | ); |
| 285 | expect(properties(complete(0, 0))).toMatchObject({ |
| 286 | usageStatus: "complete", |
| 287 | usageReported: true, |
| 288 | inputTokens: 0, |
| 289 | outputTokens: 0, |
| 290 | }); |
| 291 | const absent = properties(undefined); |
| 292 | expect(absent).toMatchObject({ usageStatus: "unavailable", usageReported: false }); |
| 293 | expect(absent).not.toHaveProperty("outputTokens"); |
| 294 | expect( |
| 295 | properties( |
| 296 | complete(0, 0, { localCommand: true, otherUsage: { inputTokens: 900, outputTokens: 80 } }), |
| 297 | ), |
| 298 | ).toMatchObject({ localCommand: true, otherInputTokens: 900, otherOutputTokens: 80 }); |
| 299 | expect(properties(complete(74_167, 236, { reasoningTokens: 30 }))).toMatchObject({ |
| 300 | outputTokens: 236, |
| 301 | reasoningTokens: 30, |
| 302 | }); |
| 303 | }); |
| 304 | |
| 305 | it("reports routed catalog models and closed efforts only", () => { |
| 306 | const routed = (model: string, variant: string) => |
| 307 | providerTurnProperties( |
| 308 | { |
| 309 | driver: "opencode", |
| 310 | context: { |
| 311 | modelSelection: { |
| 312 | instanceId: ProviderInstanceId.make("opencode"), |
| 313 | model, |
| 314 | options: [{ id: "variant", value: variant }], |
| 315 | }, |
| 316 | }, |
| 317 | providerTurn: turn("t", complete(1, 1, { models: [model] })), |
| 318 | }, |
| 319 | label, |
| 320 | ); |
| 321 | expect(routed("my-proxy/claude-opus-5-5", "high")).toMatchObject({ |
| 322 | model: "claude-opus-5-5", |
| 323 | observedModel: "claude-opus-5-5", |
| 324 | effort: "high", |
| 325 | }); |
| 326 | for (const local of [ |
| 327 | routed("ollama/acme-internal", "secret-variant"), |
| 328 | routed("acme-azure-deployment", "high"), |
| 329 | ]) { |
| 330 | expect(local).not.toHaveProperty("model"); |
| 331 | expect(local).not.toHaveProperty("observedModel"); |
| 332 | } |
| 333 | expect(routed("ollama/acme-internal", "secret-variant")).not.toHaveProperty("effort"); |
| 334 | }); |
| 335 | |
| 336 | it("reports Antigravity catalog models the bundled manifest does not list", () => { |
| 337 | expect(label("gemini-4-hydrogen")).toBe("gemini-4-hydrogen"); |
| 338 | expect(label("antigravity-default")).toBe("gemini-4-neon"); |
| 339 | expect(label("gemini-4-unlisted")).toBeUndefined(); |
| 340 | }); |
| 341 | |
| 342 | it("reports closed service tiers from a tier option or a fast mode toggle", () => { |
| 343 | const tier = (option: { id: string; value: string | boolean }) => |
| 344 | providerTurnProperties( |
| 345 | { |
| 346 | driver: "codex", |
| 347 | context: { |
| 348 | modelSelection: { |
| 349 | instanceId: ProviderInstanceId.make("codex"), |
| 350 | model: "gpt-6-sol", |
| 351 | options: [option], |
| 352 | }, |
| 353 | }, |
| 354 | providerTurn: turn("t", complete(1, 1)), |
| 355 | }, |
| 356 | label, |
| 357 | ).serviceTier; |
| 358 | expect(tier({ id: "serviceTier", value: "ultrafast" })).toBe("ultrafast"); |
| 359 | expect(tier({ id: "serviceTier", value: "default" })).toBe("standard"); |
| 360 | expect(tier({ id: "serviceTier", value: "/home/me/secret" })).toBeUndefined(); |
| 361 | expect(tier({ id: "fastMode", value: true })).toBe("fast"); |
| 362 | expect(tier({ id: "fastMode", value: false })).toBe("standard"); |
| 363 | }); |
| 364 | }); |