The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 04:45 UTC
| 1 | /** |
| 2 | * Provider token reporting: one `provider.turn.completed` per provider turn and one |
| 3 | * `provider.request.completed` per run, built from the same turns so each counts once. |
| 4 | * |
| 5 | * @module ProviderUsageAnalytics |
| 6 | */ |
| 7 | import type { |
| 8 | ModelSelection, |
| 9 | OrchestrationV2Actor, |
| 10 | OrchestrationV2CreationSource, |
| 11 | OrchestrationV2ProviderTurn, |
| 12 | OrchestrationV2RunAttempt, |
| 13 | ProviderInteractionMode, |
| 14 | RunId, |
| 15 | RuntimeMode, |
| 16 | TurnTokenUsage, |
| 17 | } from "@t3tools/contracts"; |
| 18 | import * as Context from "effect/Context"; |
| 19 | import * as DateTime from "effect/DateTime"; |
| 20 | import * as Effect from "effect/Effect"; |
| 21 | import * as Layer from "effect/Layer"; |
| 22 | |
| 23 | import { |
| 24 | getModelSelectionBooleanOptionValue, |
| 25 | getModelSelectionStringOptionValue, |
| 26 | } from "@t3tools/shared/model"; |
| 27 | import * as ProviderRegistry from "../provider/ProviderRegistry.ts"; |
| 28 | import * as AnalyticsService from "./AnalyticsService.ts"; |
| 29 | import { catalogModelLabel } from "./ProviderDimensions.ts"; |
| 30 | |
| 31 | // Bump when a field changes meaning, so reports can split old and new rows. |
| 32 | const REPORT_VERSION = 2; |
| 33 | |
| 34 | /** |
| 35 | * Where the run's starting message came from. Most turns start without a |
| 36 | * person (provider continuations, agent sends, delegated subagents), so |
| 37 | * reports need this to tell user turns apart. |
| 38 | */ |
| 39 | export interface ProviderTurnOrigin { |
| 40 | readonly createdBy: OrchestrationV2Actor; |
| 41 | readonly creationSource: OrchestrationV2CreationSource; |
| 42 | readonly scheduled: boolean; |
| 43 | readonly subagent: boolean; |
| 44 | } |
| 45 | |
| 46 | /** What started this provider turn inside its run. */ |
| 47 | export type ProviderTurnSegment = |
| 48 | | OrchestrationV2RunAttempt["reason"] |
| 49 | | "continuation" |
| 50 | | "native_child"; |
| 51 | |
| 52 | export interface ProviderTurnAnalyticsContext { |
| 53 | readonly runId?: RunId; |
| 54 | readonly modelSelection: ModelSelection; |
| 55 | readonly runtimeMode?: RuntimeMode; |
| 56 | readonly interactionMode?: ProviderInteractionMode; |
| 57 | readonly origin?: ProviderTurnOrigin; |
| 58 | readonly segment?: ProviderTurnSegment; |
| 59 | readonly restartContinuation?: boolean; |
| 60 | } |
| 61 | |
| 62 | export interface ProviderTurnReport { |
| 63 | readonly driver: string; |
| 64 | readonly providerTurn: OrchestrationV2ProviderTurn; |
| 65 | readonly context?: ProviderTurnAnalyticsContext; |
| 66 | } |
| 67 | |
| 68 | export interface RunSettledReport { |
| 69 | readonly runId: RunId; |
| 70 | readonly status: OrchestrationV2ProviderTurn["status"]; |
| 71 | readonly completedAt: DateTime.Utc; |
| 72 | // Run-owned children were still open when the run settled; their later turns are not added. |
| 73 | readonly childrenRunning: boolean; |
| 74 | } |
| 75 | |
| 76 | type Properties = Readonly<Record<string, unknown>>; |
| 77 | /** Maps a native model id to a vendor catalog label, or undefined to keep it out. */ |
| 78 | export type ModelLabeler = (model: string) => string | undefined; |
| 79 | |
| 80 | const TERMINAL_STATUSES: ReadonlySet<OrchestrationV2ProviderTurn["status"]> = new Set([ |
| 81 | "completed", |
| 82 | "failed", |
| 83 | "interrupted", |
| 84 | "cancelled", |
| 85 | ]); |
| 86 | |
| 87 | const REPORTED_EFFORTS: ReadonlySet<string> = new Set([ |
| 88 | "none", |
| 89 | "minimal", |
| 90 | "low", |
| 91 | "medium", |
| 92 | "high", |
| 93 | "xhigh", |
| 94 | "max", |
| 95 | "ultrathink", |
| 96 | "ultracode", |
| 97 | "brutal", |
| 98 | ]); |
| 99 | |
| 100 | const defined = (entries: Properties): Properties => |
| 101 | Object.fromEntries(Object.entries(entries).filter(([, value]) => value !== undefined)); |
| 102 | |
| 103 | // OpenCode names its effort option `variant`, and variants can be user-defined. |
| 104 | function selectedEffort(selection: ModelSelection): string | undefined { |
| 105 | const effort = |
| 106 | getModelSelectionStringOptionValue(selection, "reasoningEffort") ?? |
| 107 | getModelSelectionStringOptionValue(selection, "effort") ?? |
| 108 | getModelSelectionStringOptionValue(selection, "variant"); |
| 109 | return effort !== undefined && REPORTED_EFFORTS.has(effort) ? effort : undefined; |
| 110 | } |
| 111 | |
| 112 | // Codex sends a tier id (its "fast" config shows up as "priority"); Claude has a fastMode toggle. |
| 113 | const REPORTED_SERVICE_TIERS: ReadonlyMap<string, string> = new Map([ |
| 114 | ["default", "standard"], |
| 115 | ["flex", "flex"], |
| 116 | ["fast", "fast"], |
| 117 | ["priority", "fast"], |
| 118 | ["ultrafast", "ultrafast"], |
| 119 | ]); |
| 120 | |
| 121 | function selectedServiceTier(selection: ModelSelection): string | undefined { |
| 122 | const tier = getModelSelectionStringOptionValue(selection, "serviceTier"); |
| 123 | if (tier !== undefined) return REPORTED_SERVICE_TIERS.get(tier); |
| 124 | const fastMode = getModelSelectionBooleanOptionValue(selection, "fastMode"); |
| 125 | return fastMode === undefined ? undefined : fastMode ? "fast" : "standard"; |
| 126 | } |
| 127 | |
| 128 | /** |
| 129 | * Confirmed models across counted work. Unknown when any piece of work has no |
| 130 | * reported model, unless two distinct models were already seen. |
| 131 | */ |
| 132 | function modelObservation( |
| 133 | models: ReadonlyArray<ReadonlyArray<string> | undefined>, |
| 134 | label: ModelLabeler, |
| 135 | ): Properties { |
| 136 | const distinct = new Set(models.flatMap((entry) => entry ?? [])); |
| 137 | if (distinct.size > 1) return { mixedModels: true }; |
| 138 | if (distinct.size === 0 || models.some((entry) => entry === undefined || entry.length === 0)) { |
| 139 | return {}; |
| 140 | } |
| 141 | return defined({ mixedModels: false, observedModel: label([...distinct][0]!) }); |
| 142 | } |
| 143 | |
| 144 | function selectionProperties( |
| 145 | context: ProviderTurnAnalyticsContext | undefined, |
| 146 | label: ModelLabeler, |
| 147 | ) { |
| 148 | const selection = context?.modelSelection; |
| 149 | return defined({ |
| 150 | model: selection === undefined ? undefined : label(selection.model), |
| 151 | effort: selection === undefined ? undefined : selectedEffort(selection), |
| 152 | serviceTier: selection === undefined ? undefined : selectedServiceTier(selection), |
| 153 | runtimeMode: context?.runtimeMode, |
| 154 | interactionMode: context?.interactionMode, |
| 155 | ...context?.origin, |
| 156 | }); |
| 157 | } |
| 158 | |
| 159 | function childKind(context: ProviderTurnAnalyticsContext | undefined) { |
| 160 | if (context?.segment === "native_child") return "native"; |
| 161 | return context?.origin?.subagent === true ? "t3" : undefined; |
| 162 | } |
| 163 | |
| 164 | function counters(usage: TurnTokenUsage | undefined): Properties { |
| 165 | return defined({ |
| 166 | inputTokens: usage?.inputTokens, |
| 167 | cachedInputTokens: usage?.cachedInputTokens, |
| 168 | cacheCreationTokens: usage?.cacheCreationTokens, |
| 169 | outputTokens: usage?.outputTokens, |
| 170 | reasoningTokens: usage?.reasoningTokens, |
| 171 | }); |
| 172 | } |
| 173 | |
| 174 | export function providerTurnProperties( |
| 175 | report: ProviderTurnReport, |
| 176 | label: ModelLabeler, |
| 177 | ): Properties { |
| 178 | const { providerTurn, context } = report; |
| 179 | const usage = providerTurn.turnTokenUsage; |
| 180 | return defined({ |
| 181 | reportVersion: REPORT_VERSION, |
| 182 | provider: report.driver, |
| 183 | terminalStatus: providerTurn.status, |
| 184 | usageStatus: usage?.usageStatus ?? "unavailable", |
| 185 | usageScope: usage?.usageScope ?? "main_agent", |
| 186 | usageReported: usage !== undefined, |
| 187 | hasSubagents: usage?.hasSubagents, |
| 188 | ...counters(usage), |
| 189 | localCommand: usage?.localCommand === true ? true : undefined, |
| 190 | otherInputTokens: usage?.otherUsage?.inputTokens, |
| 191 | otherOutputTokens: usage?.otherUsage?.outputTokens, |
| 192 | ...selectionProperties(context, label), |
| 193 | ...modelObservation([usage?.models], label), |
| 194 | segment: context?.segment, |
| 195 | childKind: childKind(context), |
| 196 | durationMs: |
| 197 | providerTurn.startedAt && providerTurn.completedAt |
| 198 | ? Math.max( |
| 199 | 0, |
| 200 | DateTime.toEpochMillis(providerTurn.completedAt) - |
| 201 | DateTime.toEpochMillis(providerTurn.startedAt), |
| 202 | ) |
| 203 | : undefined, |
| 204 | }); |
| 205 | } |
| 206 | |
| 207 | /** How the request reached the provider; only `user` counts as a person asking. */ |
| 208 | function requestKind(origin: ProviderTurnOrigin | undefined) { |
| 209 | if (origin === undefined) return "unknown"; |
| 210 | if (origin.scheduled) return "scheduled"; |
| 211 | if (origin.subagent) return "t3_child"; |
| 212 | if (origin.createdBy === "user") return "user"; |
| 213 | if (origin.creationSource === "provider") return "provider_continuation"; |
| 214 | return origin.createdBy; |
| 215 | } |
| 216 | |
| 217 | function sumOf( |
| 218 | usages: ReadonlyArray<TurnTokenUsage>, |
| 219 | field: |
| 220 | | "inputTokens" |
| 221 | | "cachedInputTokens" |
| 222 | | "cacheCreationTokens" |
| 223 | | "outputTokens" |
| 224 | | "reasoningTokens", |
| 225 | requireAll: boolean, |
| 226 | ): number | undefined { |
| 227 | const values = usages.map((usage) => usage[field]); |
| 228 | const known = values.filter((value) => value !== undefined); |
| 229 | if (known.length === 0 || (requireAll && known.length < values.length)) return undefined; |
| 230 | return known.reduce((sum, value) => sum + value, 0); |
| 231 | } |
| 232 | |
| 233 | /** complete only when every turn is complete; partial when any turn has counts. */ |
| 234 | function combinedStatus(usages: ReadonlyArray<TurnTokenUsage | undefined>) { |
| 235 | if (usages.length > 0 && usages.every((usage) => usage?.usageStatus === "complete")) { |
| 236 | return "complete"; |
| 237 | } |
| 238 | return usages.some( |
| 239 | (usage) => usage?.inputTokens !== undefined || usage?.outputTokens !== undefined, |
| 240 | ) |
| 241 | ? "partial" |
| 242 | : "unavailable"; |
| 243 | } |
| 244 | |
| 245 | interface RequestTurn { |
| 246 | readonly child: boolean; |
| 247 | readonly segment: ProviderTurnSegment | undefined; |
| 248 | readonly providerTurn: OrchestrationV2ProviderTurn; |
| 249 | } |
| 250 | |
| 251 | interface RequestEntry { |
| 252 | driver: string | undefined; |
| 253 | context: ProviderTurnAnalyticsContext | undefined; |
| 254 | readonly lifecycleObserved: boolean; |
| 255 | readonly turns: Map<string, RequestTurn>; |
| 256 | } |
| 257 | |
| 258 | const MAX_TRACKED_RUNS = 2048; |
| 259 | |
| 260 | /** |
| 261 | * Collects terminal provider turns per run in this process. A replayed turn |
| 262 | * replaces its earlier copy, a settled run ignores late turns, and a run whose |
| 263 | * initial attempt started before this process reports `lifecycleObserved: false`. |
| 264 | */ |
| 265 | export class RequestUsageLedger { |
| 266 | private readonly open = new Map<RunId, RequestEntry>(); |
| 267 | private readonly settled = new Set<RunId>(); |
| 268 | |
| 269 | start(runId: RunId, driver: string, context: ProviderTurnAnalyticsContext): void { |
| 270 | if (this.settled.has(runId) || this.open.has(runId)) return; |
| 271 | // A restart continuation carries on a run whose earlier turns this process never saw. |
| 272 | const lifecycleObserved = context.restartContinuation !== true; |
| 273 | this.open.set(runId, { driver, context, lifecycleObserved, turns: new Map() }); |
| 274 | evictOldest(this.open); |
| 275 | } |
| 276 | |
| 277 | add(report: ProviderTurnReport): void { |
| 278 | const runId = report.context?.runId; |
| 279 | if (runId === undefined || this.settled.has(runId)) return; |
| 280 | if (!TERMINAL_STATUSES.has(report.providerTurn.status)) return; |
| 281 | let entry = this.open.get(runId); |
| 282 | if (entry === undefined) { |
| 283 | entry = { driver: undefined, context: undefined, lifecycleObserved: false, turns: new Map() }; |
| 284 | this.open.set(runId, entry); |
| 285 | evictOldest(this.open); |
| 286 | } |
| 287 | const child = report.context?.segment === "native_child"; |
| 288 | if (!child) { |
| 289 | entry.driver ??= report.driver; |
| 290 | entry.context ??= report.context; |
| 291 | } |
| 292 | entry.turns.set(String(report.providerTurn.id), { |
| 293 | child, |
| 294 | segment: report.context?.segment, |
| 295 | providerTurn: report.providerTurn, |
| 296 | }); |
| 297 | } |
| 298 | |
| 299 | settle(report: RunSettledReport, label: ModelLabeler): Properties | undefined { |
| 300 | if (this.settled.has(report.runId)) return undefined; |
| 301 | this.settled.add(report.runId); |
| 302 | evictOldest(this.settled); |
| 303 | const entry = this.open.get(report.runId); |
| 304 | this.open.delete(report.runId); |
| 305 | if (entry === undefined) return undefined; |
| 306 | |
| 307 | const turns = [...entry.turns.values()]; |
| 308 | const root = turns.filter((turn) => !turn.child); |
| 309 | const children = turns.filter((turn) => turn.child); |
| 310 | const rootUsage = root.map((turn) => turn.providerTurn.turnTokenUsage); |
| 311 | const childUsage = children.map((turn) => turn.providerTurn.turnTokenUsage); |
| 312 | const { lifecycleObserved } = entry; |
| 313 | const measuredRoot = rootUsage.filter((usage) => usage !== undefined); |
| 314 | const status = combinedStatus(rootUsage); |
| 315 | const rootComplete = status === "complete"; |
| 316 | const otherUsage = measuredRoot.map((usage) => usage.otherUsage); |
| 317 | const allOtherKnown = otherUsage.length > 0 && otherUsage.every((usage) => usage !== undefined); |
| 318 | const anyRootSubagents = measuredRoot.some((usage) => usage.hasSubagents); |
| 319 | const measuredChildren = childUsage.filter((usage) => usage !== undefined); |
| 320 | const childStatus = |
| 321 | children.length === 0 && !anyRootSubagents && !report.childrenRunning |
| 322 | ? "none" |
| 323 | : children.length === 0 |
| 324 | ? "unavailable" |
| 325 | : report.childrenRunning && combinedStatus(childUsage) === "complete" |
| 326 | ? "partial" |
| 327 | : combinedStatus(childUsage); |
| 328 | const startedAt = turns |
| 329 | .map((turn) => turn.providerTurn.startedAt) |
| 330 | .filter((value) => value !== null) |
| 331 | .map(DateTime.toEpochMillis); |
| 332 | |
| 333 | return defined({ |
| 334 | reportVersion: REPORT_VERSION, |
| 335 | provider: entry.driver ?? "unknown", |
| 336 | requestKind: requestKind(entry.context?.origin), |
| 337 | restartContinuation: entry.context?.restartContinuation === true ? true : undefined, |
| 338 | terminalStatus: report.status, |
| 339 | lifecycleObserved, |
| 340 | ...selectionProperties(entry.context, label), |
| 341 | providerTurns: root.length, |
| 342 | continuations: root.filter((turn) => turn.segment === "continuation").length, |
| 343 | steeringRestarts: root.filter((turn) => turn.segment === "steering_restart").length, |
| 344 | usageStatus: lifecycleObserved || status === "unavailable" ? status : "partial", |
| 345 | inputTokens: sumOf(measuredRoot, "inputTokens", rootComplete), |
| 346 | cachedInputTokens: sumOf(measuredRoot, "cachedInputTokens", true), |
| 347 | cacheCreationTokens: sumOf(measuredRoot, "cacheCreationTokens", true), |
| 348 | outputTokens: sumOf(measuredRoot, "outputTokens", rootComplete), |
| 349 | reasoningTokens: sumOf(measuredRoot, "reasoningTokens", true), |
| 350 | localCommand: measuredRoot.some((usage) => usage.localCommand === true) ? true : undefined, |
| 351 | otherInputTokens: allOtherKnown |
| 352 | ? otherUsage.reduce((sum, usage) => sum + (usage?.inputTokens ?? 0), 0) |
| 353 | : undefined, |
| 354 | otherOutputTokens: allOtherKnown |
| 355 | ? otherUsage.reduce((sum, usage) => sum + (usage?.outputTokens ?? 0), 0) |
| 356 | : undefined, |
| 357 | nativeChildTurns: children.length, |
| 358 | childUsageStatus: childStatus, |
| 359 | childInputTokens: sumOf(measuredChildren, "inputTokens", false), |
| 360 | childOutputTokens: sumOf(measuredChildren, "outputTokens", false), |
| 361 | childrenRunning: report.childrenRunning ? true : undefined, |
| 362 | ...modelObservation( |
| 363 | [...rootUsage, ...childUsage].map((usage) => usage?.models), |
| 364 | label, |
| 365 | ), |
| 366 | durationMs: |
| 367 | startedAt.length === 0 |
| 368 | ? undefined |
| 369 | : Math.max(0, DateTime.toEpochMillis(report.completedAt) - Math.min(...startedAt)), |
| 370 | }); |
| 371 | } |
| 372 | } |
| 373 | |
| 374 | function evictOldest(collection: Map<RunId, unknown> | Set<RunId>) { |
| 375 | if (collection.size <= MAX_TRACKED_RUNS) return; |
| 376 | const oldest = collection.keys().next().value; |
| 377 | if (oldest !== undefined) collection.delete(oldest); |
| 378 | } |
| 379 | |
| 380 | export class ProviderTurnAnalytics extends Context.Reference<{ |
| 381 | readonly runStarted: (input: { |
| 382 | readonly runId: RunId; |
| 383 | readonly driver: string; |
| 384 | readonly context: ProviderTurnAnalyticsContext; |
| 385 | }) => Effect.Effect<void>; |
| 386 | readonly turnCompleted: (report: ProviderTurnReport) => Effect.Effect<void>; |
| 387 | readonly runSettled: (report: RunSettledReport) => Effect.Effect<void>; |
| 388 | }>("t3/telemetry/ProviderTurnAnalytics", { |
| 389 | defaultValue: () => ({ |
| 390 | runStarted: () => Effect.void, |
| 391 | turnCompleted: () => Effect.void, |
| 392 | runSettled: () => Effect.void, |
| 393 | }), |
| 394 | }) {} |
| 395 | |
| 396 | export const layer = Layer.effect( |
| 397 | ProviderTurnAnalytics, |
| 398 | Effect.gen(function* () { |
| 399 | const analytics = yield* AnalyticsService.AnalyticsService; |
| 400 | const registry = yield* ProviderRegistry.ProviderRegistry; |
| 401 | const ledger = new RequestUsageLedger(); |
| 402 | const labeler = Effect.map( |
| 403 | registry.getProviders, |
| 404 | (providers): ModelLabeler => |
| 405 | (model) => |
| 406 | catalogModelLabel(providers, model), |
| 407 | ); |
| 408 | return { |
| 409 | runStarted: Effect.fn("ProviderTurnAnalytics.runStarted")((input) => |
| 410 | Effect.sync(() => ledger.start(input.runId, input.driver, input.context)), |
| 411 | ), |
| 412 | turnCompleted: Effect.fn("ProviderTurnAnalytics.turnCompleted")(function* (report) { |
| 413 | ledger.add(report); |
| 414 | yield* analytics.record( |
| 415 | "provider.turn.completed", |
| 416 | providerTurnProperties(report, yield* labeler), |
| 417 | ); |
| 418 | }), |
| 419 | runSettled: Effect.fn("ProviderTurnAnalytics.runSettled")(function* (report) { |
| 420 | const properties = ledger.settle(report, yield* labeler); |
| 421 | if (properties !== undefined) { |
| 422 | yield* analytics.record("provider.request.completed", properties); |
| 423 | } |
| 424 | }), |
| 425 | }; |
| 426 | }), |
| 427 | ); |