@devries-cameron20 / t3code-1 / commits / b92c8f4739
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): token telemetry counts each request once and only confirmed models (#29001)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Theo Browne committed Nov 1, 2026 at 18:52 UTC b92c8f473990cbc73b5320461bd06b9e26d4031b
32 files changed +1470 -178
apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts
+1
@@ -5457,6 +5457,7 @@ describe("ClaudeAdapterV2 background wake turns", () => {
5457 outputTokens: queryExits ? 40 : 46,
5458 cachedInputTokens: queryExits ? 2000 : 11455,
5459 cacheCreationTokens: queryExits ? 500 : 2847,
5460 + ...(queryExits ? {} : { models: ["claude-sonnet-4-6"] }),
5461 },
5462 );
5463 }
apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts
+47 -1
@@ -21,7 +21,14 @@ import {
21 } from "./claudeWorkflowProgress.ts";
22
23 import { isWorkspaceImagePreviewPath } from "@t3tools/shared/filePreview";
24 -import { normalizeClaudeTurnTokenUsage } from "../../provider/ClaudeTurnTokenUsage.ts";
24 +import {
25 + addClaudeLedgerUsage,
26 + claudeLedgerDelta,
27 + claudeModelLedger,
28 + normalizeClaudeTurnTokenUsage,
29 + type ClaudeLedgerUsage,
30 + type ClaudeModelLedger,
31 +} from "../../provider/ClaudeTurnTokenUsage.ts";
32 import { toMcpElicitationResponse } from "../../provider/CodexMcpElicitation.ts";
33 import {
34 type CanUseTool,
@@ -4071,6 +4078,9 @@ interface ActiveClaudeTurnContext {
4078 compactionFailure: { readonly message: string | undefined } | null;
4079 // Steers end segments whose usage is excluded from the final result; bill them too, oldest first.
4080 readonly steeredSegmentUsage: Array<SDKResultMessage["usage"]>;
4081 + // Ledger growth of the steered segments; undefined once any segment's growth is unknown.
4082 + steeredLedgerUsage: ClaudeLedgerUsage | undefined;
4083 + readonly rootModels: Set<string>;
4084 readonly subagentsByTaskId: Map<string, ActiveClaudeSubagent>;
4085 readonly subagentsByToolUseId: Map<string, ActiveClaudeSubagent>;
4086 readonly promptUuid: string | null;
@@ -4122,6 +4132,8 @@ interface ClaudeLiveQueryContext {
4132 permissionMode: PermissionMode;
4133 stopping: boolean;
4134 readonly subagentsFromEarlierProcesses: ReadonlySet<ActiveClaudeSubagent>;
4135 + // The SDK's per-process model ledger at the last result; a new process starts at zero.
4136 + modelLedger: ClaudeModelLedger | undefined;
4137 // Per process, not per session: a relative temp dir follows the cwd this CLI
4138 // was launched in, and a later turn's cwd can differ.
4139 readonly taskOutputRoot: string | undefined;
@@ -4775,6 +4787,8 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
4787 const interruptedTurns = yield* Ref.make(new Set<OrchestrationV2ProviderTurn["id"]>());
4788 const steeredTurns = yield* Ref.make(new Set<OrchestrationV2ProviderTurn["id"]>());
4789 const queryContext = yield* Ref.make<ClaudeLiveQueryContext | null>(null);
4790 + // Wake buffers replay result frames, so ledger growth is measured once on arrival.
4791 + const resultLedgerUsage = new WeakMap<SDKMessage, ClaudeLedgerUsage>();
4792 // A worktree handoff changes the project settings even within one session.
4793 let bypassCache: { cwd: string | null; availability: ClaudeBypassAvailability } | null =
4794 null;
@@ -7613,6 +7627,17 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
7627 input.context.subagentsByToolUseId.size > 0,
7628 input.status,
7629 input.context.steeredSegmentUsage,
7630 + {
7631 + rootModels: input.context.rootModels,
7632 + ...(input.result === undefined
7633 + ? {}
7634 + : {
7635 + ledger: addClaudeLedgerUsage(
7636 + input.context.steeredLedgerUsage,
7637 + resultLedgerUsage.get(input.result),
7638 + ),
7639 + }),
7640 + },
7641 ),
7642 },
7643 }),
@@ -8528,6 +8553,10 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
8553
8554 if (message.type === "assistant" && input.replayed !== true) {
8555 if (message.parent_tool_use_id === null) {
8556 + // CLI notices and API errors arrive as assistant messages from "<synthetic>".
8557 + if (message.message.model && message.message.model !== "<synthetic>") {
8558 + context.rootModels.add(message.message.model);
8559 + }
8560 // Subagent uuids exist only in the sidechain transcript, so resume and fork cannot target them.
8561 context.nativeMessageCursor = message.uuid;
8562 context.latestAssistantRateLimit =
@@ -9409,6 +9438,10 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
9438 const wasSteered = (yield* Ref.get(steeredTurns)).has(context.providerTurnId);
9439 if (!interrupted && wasSteered && isClaudeActiveSteeringAbortResult(message)) {
9440 context.steeredSegmentUsage.push(message.usage);
9441 + context.steeredLedgerUsage = addClaudeLedgerUsage(
9442 + context.steeredLedgerUsage,
9443 + resultLedgerUsage.get(message),
9444 + );
9445 return;
9446 }
9447 yield* Ref.update(steeredTurns, (current) => {
@@ -10404,6 +10437,7 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
10437 (subagent) => subagent.task.status === "running",
10438 ),
10439 ),
10440 + modelLedger: new Map(),
10441 taskOutputRoot: claudeTaskOutputRoot(
10442 adapterOptions.environment,
10443 path,
@@ -10432,6 +10466,16 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
10466 ) {
10467 context.permissionMode = message.permissionMode;
10468 }
10469 + if (message.type === "result") {
10470 + const ledger = claudeModelLedger(message.modelUsage);
10471 + const growth = claudeLedgerDelta(context.modelLedger, ledger);
10472 + if (growth !== undefined) resultLedgerUsage.set(message, growth);
10473 + // A ledger that went backwards cannot anchor the next turn's growth.
10474 + context.modelLedger =
10475 + growth === undefined && context.modelLedger !== undefined
10476 + ? undefined
10477 + : ledger;
10478 + }
10479 return handleSdkMessage({ query: querySession, message });
10480 }),
10481 ),
@@ -10577,6 +10621,8 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* (
10621 latestAssistantRateLimit: null,
10622 compactionFailure: null,
10623 steeredSegmentUsage: [],
10624 + steeredLedgerUsage: { inputTokens: 0, outputTokens: 0 },
10625 + rootModels: new Set(),
10626 subagentsByTaskId: new Map(),
10627 subagentsByToolUseId: new Map(),
10628 promptUuid,
apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts
+28 -6
@@ -1830,6 +1830,17 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
1830 const turnWaiters = yield* Ref.make(new Map<string, Deferred.Deferred<void, never>>());
1831 const subagentThreads = yield* Ref.make(new Map<string, CodexSubagentThreadContext>());
1832 const subagentSelections = new Map<string, Omit<ModelSelection, "instanceId">>();
1833 + // A reroute names one turn; until new settings arrive, the thread's model is unknown.
1834 + const reroutedTurnModels = new Map<string, string>();
1835 + const reroutedThreads = new Set<string>();
1836 + const reportedTurnModel = (nativeThreadId: string, nativeTurnId: string) => {
1837 + const rerouted = reroutedTurnModels.get(nativeTurnId);
1838 + reroutedTurnModels.delete(nativeTurnId);
1839 + if (rerouted !== undefined) return rerouted;
1840 + return reroutedThreads.has(nativeThreadId)
1841 + ? undefined
1842 + : subagentSelections.get(nativeThreadId)?.model;
1843 + };
1844 const pendingSubagentTurns = yield* Ref.make(
1845 new Map<string, ReadonlyArray<PendingCodexSubagentTurnStarted>>(),
1846 );
@@ -4475,15 +4486,22 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
4486 );
4487
4488 yield* client.handleServerNotification("thread/settings/updated", (payload) =>
4478 - updateSubagentSelection(
4479 - payload.threadId,
4480 - payload.threadSettings.model,
4481 - payload.threadSettings.effort,
4482 - payload.threadSettings.serviceTier,
4489 + Effect.sync(() => reroutedThreads.delete(payload.threadId)).pipe(
4490 + Effect.andThen(
4491 + updateSubagentSelection(
4492 + payload.threadId,
4493 + payload.threadSettings.model,
4494 + payload.threadSettings.effort,
4495 + payload.threadSettings.serviceTier,
4496 + ),
4497 + ),
4498 ),
4499 );
4500 yield* client.handleServerNotification("model/rerouted", (payload) =>
4486 - updateSubagentSelection(payload.threadId, payload.toModel),
4501 + Effect.sync(() => {
4502 + reroutedTurnModels.set(payload.turnId, payload.toModel);
4503 + reroutedThreads.add(payload.threadId);
4504 + }).pipe(Effect.andThen(updateSubagentSelection(payload.threadId, payload.toModel))),
4505 );
4506
4507 yield* client.handleServerNotification("hook/completed", (payload) =>
@@ -6127,6 +6145,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
6145 ),
6146 input.nativeTurnId,
6147 input.status === "completed",
6148 + reportedTurnModel(
6149 + input.context.providerThread.nativeThreadRef?.nativeId ?? "",
6150 + input.nativeTurnId,
6151 + ),
6152 ),
6153 };
6154 if (!holdsForGoal) {
apps/server/src/orchestration-v2/Adapters/OpenCode2AdapterV2.test.ts
+57
@@ -2150,6 +2150,20 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => {
2150 const { runtime, thread } = yield* resumed([
2151 out("session.prompt", { sessionID: SESSION, id: PROMPT_ID, text: "<any>" }),
2152 promptAccepted,
2153 + event("session.step.started", {
2154 + sessionID: SESSION,
2155 + assistantMessageID: "msg_before_gap",
2156 + agent: "build",
2157 + model: { id: "big-pickle", providerID: "opencode", variant: "default" },
2158 + started: 1,
2159 + }),
2160 + event("session.step.ended", {
2161 + sessionID: SESSION,
2162 + assistantMessageID: "msg_before_gap",
2163 + finish: "tool-calls",
2164 + cost: 0,
2165 + tokens: { input: 100, output: 10, reasoning: 0, cache: { read: 0, write: 0 } },
2166 + }),
2167 ...reconnected({}, "succeeded"),
2168 ]);
2169 const events = yield* runtime.events.pipe(
@@ -2164,6 +2178,15 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => {
2178 "command_execution:completed",
2179 ]);
2180 assert.deepInclude(collected.at(-1), { type: "turn.terminal", status: "completed" });
2181 + // Steps that ended during the gap sent no tokens or models, so both stay incomplete.
2182 + const settled = collected.findLast(
2183 + (event) =>
2184 + event.type === "provider_turn.updated" && event.providerTurn.completedAt !== null,
2185 + );
2186 + const usage =
2187 + settled?.type === "provider_turn.updated" ? settled.providerTurn.turnTokenUsage : undefined;
2188 + assert.deepInclude(usage, { usageStatus: "partial", inputTokens: 100, outputTokens: 10 });
2189 + assert.notProperty(usage, "models");
2190 }).pipe(Effect.scoped),
2191 );
2192
@@ -2879,6 +2902,40 @@ it.layer(McpProviderSessions.layer)("OpenCode2 adapter", (it) => {
2902 }).pipe(Effect.scoped),
2903 );
2904
2905 + it.effect("leaves models unknown when a step ends without a seen start", () =>
2906 + Effect.gen(function* () {
2907 + const tokens = { input: 10, output: 1, reasoning: 0, cache: { read: 0, write: 0 } };
2908 + const step = (id: string) => ({ sessionID: SESSION, assistantMessageID: id });
2909 + const { runtime, thread } = yield* resumed([
2910 + out("session.prompt", { sessionID: SESSION, text: "<any>" }),
2911 + promptAccepted,
2912 + event("session.step.started", {
2913 + ...step("msg_seen"),
2914 + agent: "build",
2915 + model: { id: "big-pickle", providerID: "opencode", variant: "default" },
2916 + started: 1,
2917 + }),
2918 + event("session.step.ended", { ...step("msg_seen"), finish: "tool-calls", cost: 0, tokens }),
2919 + event("session.step.ended", { ...step("msg_unseen"), finish: "stop", cost: 0, tokens }),
2920 + event("session.execution.succeeded", { sessionID: SESSION }),
2921 + ]);
2922 + const turn = yield* runtime.events.pipe(
2923 + Stream.filter(
2924 + (event) =>
2925 + event.type === "provider_turn.updated" && event.providerTurn.completedAt !== null,
2926 + ),
2927 + Stream.runHead,
2928 + Effect.forkScoped,
2929 + );
2930 + yield* runtime.startTurn(turnInput(thread));
2931 + const settled = Option.getOrUndefined(yield* Fiber.join(turn));
2932 + const usage =
2933 + settled?.type === "provider_turn.updated" ? settled.providerTurn.turnTokenUsage : undefined;
2934 + assert.deepInclude(usage, { usageStatus: "complete", inputTokens: 20, outputTokens: 2 });
2935 + assert.notProperty(usage, "models");
2936 + }).pipe(Effect.scoped),
2937 + );
2938 +
2939 it.effect("reports cache writes as cache creation, not only as input", () =>
2940 Effect.gen(function* () {
2941 const { runtime, thread } = yield* resumed([
apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts
+10 -21
@@ -45,6 +45,7 @@ import * as EventStore from "./EventStore.ts";
45 import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
46 import * as ProjectionStore from "./ProjectionStore.ts";
47 import * as ProviderEventIngestor from "./ProviderEventIngestor.ts";
48 +import * as ProviderUsageAnalytics from "../telemetry/ProviderUsageAnalytics.ts";
49 import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
50 import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.ts";
51 import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
@@ -325,12 +326,14 @@ it.effect("refreshes the failing instance snapshot after a committed usage_limit
326 );
327
328 it.effect("records accepted billed turn usage once without billing the context window", () => {
328 - const recorded: Array<Readonly<Record<string, unknown>>> = [];
329 - const layerAnalytics = Layer.succeed(ProviderEventIngestor.ProviderTurnAnalytics, {
330 - record: (properties: Readonly<Record<string, unknown>>) =>
329 + const recorded: Array<ProviderUsageAnalytics.ProviderTurnReport> = [];
330 + const layerAnalytics = Layer.succeed(ProviderUsageAnalytics.ProviderTurnAnalytics, {
331 + runStarted: () => Effect.void,
332 + turnCompleted: (report) =>
333 Effect.sync(() => {
332 - recorded.push(properties);
334 + recorded.push(report);
335 }),
336 + runSettled: () => Effect.void,
337 });
338 return Effect.gen(function* () {
339 const now = yield* DateTime.now;
@@ -413,23 +416,9 @@ it.effect("records accepted billed turn usage once without billing the context w
416 assert.isEmpty(ignored);
417 assert.lengthOf(recorded, 1);
418 assert.deepEqual(recorded[0], {
416 - provider: CODEX_DRIVER,
417 - terminalStatus: "completed",
418 - usageStatus: "complete",
419 - usageScope: "main_agent",
420 - hasSubagents: false,
421 - inputTokens: 40,
422 - cachedInputTokens: 30,
423 - outputTokens: 10,
424 - model: modelSelection.model,
425 - mixedModels: false,
426 - runtimeMode: "full-access",
427 - interactionMode: "default",
428 - createdBy: "agent",
429 - creationSource: "provider",
430 - scheduled: false,
431 - subagent: true,
432 - durationMs: 120,
419 + driver: CODEX_DRIVER,
420 + providerTurn,
421 + context: input.analyticsContext,
422 });
423 }).pipe(Effect.provide(layerTest.pipe(Layer.provide(layerAnalytics))));
424 });
apps/server/src/orchestration-v2/ProviderEventIngestor.ts
+12 -98
@@ -5,14 +5,9 @@ import {
5 OrchestrationV2StoredEvent,
6 type OrchestrationV2PlanArtifact,
7 type OrchestrationV2Run,
8 - type OrchestrationV2ProviderTurn,
8 type OrchestrationV2Subagent,
9 type OrchestrationV2ThreadProjection,
10 type ModelSelection,
12 - type OrchestrationV2Actor,
13 - type OrchestrationV2CreationSource,
14 - type RuntimeMode,
15 - type ProviderInteractionMode,
11 ProviderInstanceId,
12 ProviderSessionId,
13 ProviderThreadId,
@@ -31,8 +26,11 @@ import * as SqlError from "effect/sql/SqlError";
26
27 import { AgentScope } from "@t3tools/shared/AgentScope";
28 import { isTemporaryWorktreeBranch } from "@t3tools/shared/git";
34 -import { getModelSelectionStringOptionValue, modelSelectionsEqual } from "@t3tools/shared/model";
35 -import * as AnalyticsService from "../telemetry/AnalyticsService.ts";
29 +import { modelSelectionsEqual } from "@t3tools/shared/model";
30 +import {
31 + ProviderTurnAnalytics,
32 + type ProviderTurnAnalyticsContext,
33 +} from "../telemetry/ProviderUsageAnalytics.ts";
34 import * as GitVcsDriver from "../vcs/GitVcsDriver.ts";
35 import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.ts";
36 import * as EventSink from "./EventSink.ts";
@@ -85,88 +83,6 @@ export const ProviderEventIngestorV2Error = Schema.Union([
83 ]);
84 export type ProviderEventIngestorV2Error = typeof ProviderEventIngestorV2Error.Type;
85
88 -export interface ProviderTurnAnalyticsContext {
89 - readonly modelSelection: ModelSelection;
90 - readonly runtimeMode?: RuntimeMode;
91 - readonly interactionMode?: ProviderInteractionMode;
92 - readonly origin?: ProviderTurnOrigin;
93 -}
94 -
95 -/**
96 - * Where the run's starting message came from. Most turns start without a
97 - * person (provider continuations, agent sends, delegated subagents), so
98 - * reports need this to tell user turns apart.
99 - */
100 -export interface ProviderTurnOrigin {
101 - readonly createdBy: OrchestrationV2Actor;
102 - readonly creationSource: OrchestrationV2CreationSource;
103 - readonly scheduled: boolean;
104 - readonly subagent: boolean;
105 -}
106 -
107 -export class ProviderTurnAnalytics extends Context.Reference<{
108 - readonly record: (properties: Readonly<Record<string, unknown>>) => Effect.Effect<void>;
109 -}>("t3/orchestration-v2/ProviderTurnAnalytics", {
110 - defaultValue: () => ({ record: () => Effect.void }),
111 -}) {}
112 -
113 -export const layerAnalytics = Layer.effect(
114 - ProviderTurnAnalytics,
115 - Effect.gen(function* () {
116 - const analytics = yield* AnalyticsService.AnalyticsService;
117 - return {
118 - record: Effect.fn("ProviderTurnAnalytics.record")(function (
119 - properties: Readonly<Record<string, unknown>>,
120 - ) {
121 - return analytics.record("provider.turn.completed", properties);
122 - }),
123 - };
124 - }),
125 -);
126 -
127 -function providerTurnAnalyticsProperties(input: {
128 - readonly driver: ProviderAdapter.ProviderAdapterV2Event["driver"];
129 - readonly providerTurn: OrchestrationV2ProviderTurn;
130 - readonly context?: ProviderTurnAnalyticsContext;
131 -}): Readonly<Record<string, unknown>> {
132 - const usage = input.providerTurn.turnTokenUsage;
133 - const modelSelection = input.context?.modelSelection;
134 - const effort = modelSelection
135 - ? (getModelSelectionStringOptionValue(modelSelection, "reasoningEffort") ??
136 - getModelSelectionStringOptionValue(modelSelection, "effort"))
137 - : undefined;
138 - return {
139 - provider: input.driver,
140 - terminalStatus: input.providerTurn.status,
141 - usageStatus: usage?.usageStatus ?? "unavailable",
142 - usageScope: usage?.usageScope ?? "main_agent",
143 - ...(usage ? { hasSubagents: usage.hasSubagents } : {}),
144 - ...(usage?.inputTokens === undefined ? {} : { inputTokens: usage.inputTokens }),
145 - ...(usage?.cachedInputTokens === undefined
146 - ? {}
147 - : { cachedInputTokens: usage.cachedInputTokens }),
148 - ...(usage?.cacheCreationTokens === undefined
149 - ? {}
150 - : { cacheCreationTokens: usage.cacheCreationTokens }),
151 - ...(usage?.outputTokens === undefined ? {} : { outputTokens: usage.outputTokens }),
152 - ...(usage?.reasoningTokens === undefined ? {} : { reasoningTokens: usage.reasoningTokens }),
153 - ...(modelSelection ? { model: modelSelection.model, mixedModels: false } : {}),
154 - ...(effort ? { effort } : {}),
155 - ...(input.context?.runtimeMode ? { runtimeMode: input.context.runtimeMode } : {}),
156 - ...(input.context?.interactionMode ? { interactionMode: input.context.interactionMode } : {}),
157 - ...input.context?.origin,
158 - ...(input.providerTurn.startedAt && input.providerTurn.completedAt
159 - ? {
160 - durationMs: Math.max(
161 - 0,
162 - DateTime.toEpochMillis(input.providerTurn.completedAt) -
163 - DateTime.toEpochMillis(input.providerTurn.startedAt),
164 - ),
165 - }
166 - : {}),
167 - };
168 -}
169 -
86 type TodoListPlan = Extract<OrchestrationV2PlanArtifact, { readonly kind: "todo_list" }>;
87
88 function withPlanStepDurations(
@@ -891,15 +807,13 @@ export const layer: Layer.Layer<
807 const oldest = completedTurnAnalytics.values().next().value;
808 if (oldest !== undefined) completedTurnAnalytics.delete(oldest);
809 }
894 - yield* analytics.record(
895 - providerTurnAnalyticsProperties({
896 - driver: input.event.driver,
897 - providerTurn,
898 - ...(input.analyticsContext === undefined
899 - ? {}
900 - : { context: input.analyticsContext }),
901 - }),
902 - );
810 + yield* analytics.turnCompleted({
811 + driver: input.event.driver,
812 + providerTurn,
813 + ...(input.analyticsContext === undefined
814 + ? {}
815 + : { context: input.analyticsContext }),
816 + });
817 }),
818 ),
819 (effect) =>
apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts
+10
@@ -51,6 +51,7 @@ import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"
51 import * as ProviderSessionManager from "./ProviderSessionManager.ts";
52 import * as ProviderTurnStart from "./ProviderTurnStartService.ts";
53 import * as RunExecutionService from "./RunExecutionService.ts";
54 +import * as ProviderUsageAnalytics from "../telemetry/ProviderUsageAnalytics.ts";
55 import * as RuntimePolicy from "./RuntimePolicy.ts";
56
57 const isDomainEvent = Schema.is(OrchestrationV2DomainEvent);
@@ -301,6 +302,7 @@ function makeLocalCommandHarness(input: {
302 readonly switchedProvider?: boolean;
303 };
304 }) {
305 + const settledRuns: Array<string> = [];
306 const now = DateTime.makeUnsafe("2026-09-04T12:00:00Z");
307 const threadId = ThreadId.make("thread-native-account-command");
308 const runId = RunId.make("run-native-account-command");
@@ -947,6 +949,11 @@ function makeLocalCommandHarness(input: {
949 }),
950 Layer.mock(ProviderAuthService.ProviderAuthService)({ tryHandlePromptCommand }),
951 Layer.mock(RunExecutionService.RunExecutionServiceV2)({ startRootRun }),
952 + Layer.succeed(ProviderUsageAnalytics.ProviderTurnAnalytics, {
953 + runStarted: () => Effect.void,
954 + turnCompleted: () => Effect.void,
955 + runSettled: (report) => Effect.sync(() => void settledRuns.push(report.status)),
956 + }),
957 Layer.mock(RuntimePolicy.RuntimePolicyV2)({
958 resolve: resolveRuntimePolicy,
959 }),
@@ -954,6 +961,7 @@ function makeLocalCommandHarness(input: {
961 ),
962 );
963 return {
964 + settledRuns,
965 getThreadRecords,
966 getTurnStartContext,
967 setProjection: (next: OrchestrationV2ThreadProjection) => {
@@ -1578,6 +1586,7 @@ effectIt.effect("terminalizes a starting run when its provider session cannot op
1586 );
1587 const projection = harness.projection();
1588 expect(projection.runs.at(-1)).toMatchObject({ status: "failed", startedAt: null });
1589 + expect(harness.settledRuns).toEqual(["failed"]);
1590 expect(projection.attempts[0]).toMatchObject({ status: "failed", startedAt: null });
1591 expect(projection.nodes[0]).toMatchObject({ status: "failed", startedAt: null });
1592 expect(projection.turnItems).toMatchObject([
@@ -1667,6 +1676,7 @@ effectIt.effect("does not overwrite a run interrupted while its provider session
1676 expect(harness.startRootRun).not.toHaveBeenCalled();
1677 const projection = harness.projection();
1678 expect(projection.runs.at(-1)?.status).toBe("interrupted");
1679 + expect(harness.settledRuns).toEqual([]);
1680 expect(projection.attempts[0]?.status).toBe("pending");
1681 expect(projection.nodes[0]?.status).toBe("pending");
1682 expect(projection.turnItems).toEqual([]);
apps/server/src/orchestration-v2/ProviderTurnStartService.ts
+12 -1
@@ -33,6 +33,7 @@ import * as ProviderAuthService from "../provider/ProviderAuthService.ts";
33 import * as ServerSettings from "../serverSettings.ts";
34 import { makeIsManagedWorktreePath } from "../worktreesDirectory.ts";
35 import * as EventSink from "./EventSink.ts";
36 +import * as ProviderUsageAnalytics from "../telemetry/ProviderUsageAnalytics.ts";
37 import * as ContextHandoffService from "./ContextHandoffService.ts";
38 import { isActiveRun } from "./CommandPolicy.ts";
39 import {
@@ -148,6 +149,7 @@ export const layer: Layer.Layer<
149 ProviderTurnStartServiceV2,
150 Effect.gen(function* () {
151 const eventSink = yield* EventSink.EventSinkV2;
152 + const turnAnalytics = yield* ProviderUsageAnalytics.ProviderTurnAnalytics;
153 const contextHandoffService = yield* ContextHandoffService.ContextHandoffServiceV2;
154 const idAllocator = yield* IdAllocator.IdAllocatorV2;
155 const fileSystem = yield* FileSystem.FileSystem;
@@ -496,13 +498,22 @@ export const layer: Layer.Layer<
498 } satisfies OrchestrationV2DomainEvent;
499 }),
500 );
499 - yield* eventSink.writeIfRunCurrent({
501 + const written = yield* eventSink.writeIfRunCurrent({
502 threadId: projection.thread.id,
503 runId,
504 activeAttemptId: attempt.id,
505 expectedStatus: "starting",
506 events,
507 });
508 + // A steering restart that cannot start still ends a run whose earlier turns counted.
509 + if (written.committed) {
510 + yield* turnAnalytics.runSettled({
511 + runId,
512 + status,
513 + completedAt: now,
514 + childrenRunning: false,
515 + });
516 + }
517 },
518 );
519 const settleStartFailure = Effect.fn("ProviderTurnStartService.settleStartFailure")(
apps/server/src/orchestration-v2/RunExecutionService.test.ts
+26 -3
@@ -47,6 +47,7 @@ import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
47 import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
48 import * as ProjectionStore from "./ProjectionStore.ts";
49 import * as ProviderEventIngestor from "./ProviderEventIngestor.ts";
50 +import * as ProviderUsageAnalytics from "../telemetry/ProviderUsageAnalytics.ts";
51 import * as McpAppModelContext from "../mcpApps/McpAppModelContext.ts";
52 import * as PreviewAgentActivity from "../preview/PreviewAgentActivity.ts";
53 import * as RunExecutionService from "./RunExecutionService.ts";
@@ -1413,10 +1414,21 @@ it.effect("ingests new member threads of an owned child after the root turn term
1414 const subagentNodeId = NodeId.make("node:run-execution-late-child:subagent");
1415 const childMessageIngested = yield* Deferred.make<void>();
1416 const order = yield* Ref.make<ReadonlyArray<string>>([]);
1416 - const origins = yield* Ref.make<ReadonlyArray<readonly [string, unknown]>>([]);
1417 + const origins = yield* Ref.make<ReadonlyArray<readonly [string, unknown, unknown]>>([]);
1418 + const requests = yield* Ref.make<ReadonlyArray<string>>([]);
1419 const layerTest = RunExecutionService.layer.pipe(
1420 Layer.provide(
1421 Layer.mergeAll(
1422 + Layer.succeed(ProviderUsageAnalytics.ProviderTurnAnalytics, {
1423 + runStarted: (input) =>
1424 + Ref.update(requests, (current) => [...current, `started:${input.runId}`]),
1425 + turnCompleted: () => Effect.void,
1426 + runSettled: (input) =>
1427 + Ref.update(requests, (current) => [
1428 + ...current,
1429 + `settled:${input.runId}:${input.status}:children-running=${input.childrenRunning}`,
1430 + ]),
1431 + }),
1432 McpAppModelContext.layerEmpty,
1433 PreviewAgentActivity.layer,
1434 Layer.mock(CheckpointService.CheckpointServiceV2)({ captureBaseline: () => Effect.void }),
@@ -1443,7 +1455,11 @@ it.effect("ingests new member threads of an owned child after the root turn term
1455 const eventType = input.event.type;
1456 yield* Ref.update(origins, (current) => [
1457 ...current,
1446 - [eventType, input.analyticsContext?.origin] as const,
1458 + [
1459 + eventType,
1460 + input.analyticsContext?.origin,
1461 + input.analyticsContext?.segment,
1462 + ] as const,
1463 ]);
1464 if (
1465 input.event.type === "app_thread.created" &&
@@ -1603,6 +1619,7 @@ it.effect("ingests new member threads of an owned child after the root turn term
1619 attempt: {
1620 id: attemptId,
1621 providerTurnId: rootProviderTurnId,
1622 + reason: "initial",
1623 } as OrchestrationV2RunAttempt,
1624 attemptId,
1625 providerTurnOrdinal: 1,
@@ -1635,7 +1652,7 @@ it.effect("ingests new member threads of an owned child after the root turn term
1652 assert.deepEqual(yield* Ref.get(order), ["root-finalized", "member-thread", "child-message"]);
1653 // The run's message started the root; the parent agent started the child's turn.
1654 const recorded = yield* Ref.get(origins);
1638 - assert.deepEqual(recorded[0], [
1655 + assert.deepEqual(recorded[0]?.slice(0, 2), [
1656 "app_thread.created",
1657 { createdBy: "user", creationSource: "web", scheduled: false, subagent: false },
1658 ]);
@@ -1644,8 +1661,14 @@ it.effect("ingests new member threads of an owned child after the root turn term
1661 [
1662 "provider_turn.updated",
1663 { createdBy: "agent", creationSource: "provider", scheduled: false, subagent: true },
1664 + "native_child",
1665 ],
1666 );
1667 + // The child outlives the root, so the request summary says its later turns are missing.
1668 + assert.deepEqual(yield* Ref.get(requests), [
1669 + `started:${runId}`,
1670 + `settled:${runId}:completed:children-running=true`,
1671 + ]);
1672 }),
1673 );
1674
apps/server/src/orchestration-v2/RunExecutionService.ts
+49 -12
@@ -46,6 +46,7 @@ import * as EventSink from "./EventSink.ts";
46 import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
47 import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
48 import * as ProviderEventIngestor from "./ProviderEventIngestor.ts";
49 +import * as ProviderUsageAnalytics from "../telemetry/ProviderUsageAnalytics.ts";
50 import type { ProjectionStoreV2Error } from "./ProjectionStore.ts";
51 import {
52 knownProviderFailureMessage,
@@ -98,7 +99,7 @@ function isSettledSubagentStatus(status: OrchestrationV2Subagent["status"]): boo
99 }
100
101 /** A provider-native subagent's turns arrive on its parent's run, but the parent agent started them. */
101 -const NATIVE_SUBAGENT_TURN_ORIGIN: ProviderEventIngestor.ProviderTurnOrigin = {
102 +const NATIVE_SUBAGENT_TURN_ORIGIN: ProviderUsageAnalytics.ProviderTurnOrigin = {
103 createdBy: "agent",
104 creationSource: "provider",
105 scheduled: false,
@@ -615,6 +616,7 @@ export const layer: Layer.Layer<
616 const previewAgentActivity = yield* PreviewAgentActivity.PreviewAgentActivity;
617 const turnItemPositions = yield* TurnItemPositionStore.TurnItemPositionStoreV2;
618 const finalizationObserver = yield* RunFinalizationService.RunFinalizationObserver;
619 + const turnAnalytics = yield* ProviderUsageAnalytics.ProviderTurnAnalytics;
620
621 const writeFinalRunEvents = Effect.fn("RunExecutionService.writeFinalRunEvents")(
622 function* (input: {
@@ -871,13 +873,19 @@ export const layer: Layer.Layer<
873 } else {
874 yield* eventSink.writeWithEffects(finalization);
875 }
876 + yield* turnAnalytics.runSettled({
877 + runId: input.run.id,
878 + status: input.terminal.status,
879 + completedAt,
880 + childrenRunning: hasOpenSubagentProjection && cascadedSubagentEvents.length === 0,
881 + });
882 yield* input.refreshAfterTurn;
883 },
884 );
885
886 return RunExecutionServiceV2.of({
887 startRootRun: Effect.fn("RunExecutionService.startRootRun")(function* (input) {
880 - const rootTurnOrigin: ProviderEventIngestor.ProviderTurnOrigin = {
888 + const rootTurnOrigin: ProviderUsageAnalytics.ProviderTurnOrigin = {
889 createdBy: input.message.createdBy,
890 creationSource: input.message.creationSource,
891 scheduled: input.message.scheduledTaskId !== undefined,
@@ -997,6 +1005,25 @@ export const layer: Layer.Layer<
1005 const latestTurnItemOrdinal = yield* Ref.make(input.providerTurnOrdinal * 100);
1006 const latestProviderThread = yield* Ref.make(input.providerThread);
1007 const latestRootProviderTurn = yield* Ref.make<OrchestrationV2ProviderTurn | null>(null);
1008 + // Later root turns in one attempt are provider continuations, such as Codex goal turns.
1009 + let firstRootProviderTurnId: ProviderTurnId | undefined;
1010 + const analyticsContext = {
1011 + runId: input.run.id,
1012 + modelSelection: input.modelSelection,
1013 + runtimeMode: input.runtimePolicy.runtimeMode,
1014 + interactionMode: input.runtimePolicy.interactionMode,
1015 + origin: rootTurnOrigin,
1016 + ...(input.run.restartContinuationOfRunId === undefined
1017 + ? {}
1018 + : { restartContinuation: true }),
1019 + } satisfies ProviderUsageAnalytics.ProviderTurnAnalyticsContext;
1020 + if (input.attempt.reason === "initial") {
1021 + yield* turnAnalytics.runStarted({
1022 + runId: input.run.id,
1023 + driver: input.providerThread.driver,
1024 + context: { ...analyticsContext, segment: "initial" },
1025 + });
1026 + }
1027 const routeIdentity: ProviderEventRouteIdentity = {
1028 threadId: input.run.threadId,
1029 runId: input.run.id,
@@ -1396,17 +1423,27 @@ export const layer: Layer.Layer<
1423 const isRootProviderThreadUpdate =
1424 event.type === "provider_thread.updated" &&
1425 event.providerThread.id === input.providerThread.id;
1426 + const nativeChildTurn =
1427 + event.type === "provider_turn.updated" &&
1428 + event.providerTurn.providerThreadId !== input.providerThread.id;
1429 + if (event.type === "provider_turn.updated" && !nativeChildTurn) {
1430 + firstRootProviderTurnId ??= event.providerTurn.id;
1431 + }
1432 const storedEvents = yield* providerEventIngestor.ingestNormalized({
1400 - analyticsContext: {
1401 - modelSelection: input.modelSelection,
1402 - runtimeMode: input.runtimePolicy.runtimeMode,
1403 - interactionMode: input.runtimePolicy.interactionMode,
1404 - origin:
1405 - event.type === "provider_turn.updated" &&
1406 - event.providerTurn.providerThreadId !== input.providerThread.id
1407 - ? NATIVE_SUBAGENT_TURN_ORIGIN
1408 - : rootTurnOrigin,
1409 - },
1433 + analyticsContext: nativeChildTurn
1434 + ? {
1435 + ...analyticsContext,
1436 + origin: NATIVE_SUBAGENT_TURN_ORIGIN,
1437 + segment: "native_child",
1438 + }
1439 + : {
1440 + ...analyticsContext,
1441 + segment:
1442 + event.type === "provider_turn.updated" &&
1443 + event.providerTurn.id !== firstRootProviderTurnId
1444 + ? "continuation"
1445 + : input.attempt.reason,
1446 + },
1447 providerSessionId: input.providerSessionId,
1448 providerInstanceId: input.run.providerInstanceId,
1449 threadId: input.run.threadId,
apps/server/src/orchestration-v2/testkit/fixtures/claude_compact_after_peer_turn/output.ts
+21
@@ -51,4 +51,25 @@ export function assertClaudeCompactAfterPeerTurnOutput(
51 assert.equal(runs[2]?.status, "completed");
52 assert.deepEqual(runs[2]?.replies, ["PEER_ACK"]);
53 assert.deepEqual(runs[2]?.compactions, []);
54 +
55 + const usage = projection.runs.map(
56 + (run) =>
57 + projection.providerTurns.find((turn) => turn.runAttemptId === run.activeAttemptId)
58 + ?.turnTokenUsage,
59 + );
60 + assert.deepInclude(usage[0], {
61 + outputTokens: 7,
62 + otherUsage: { inputTokens: 1015, outputTokens: 12 },
63 + });
64 + // /compact reports zero tokens as a local command.
65 + assert.deepInclude(usage[1], {
66 + usageStatus: "complete",
67 + inputTokens: 0,
68 + outputTokens: 0,
69 + localCommand: true,
70 + });
71 + // The peer result's ledger is empty, so neither it nor the next result has a baseline.
72 + assert.deepInclude(usage[2], { usageStatus: "complete", outputTokens: 4 });
73 + assert.notProperty(usage[1], "otherUsage");
74 + assert.notProperty(usage[2], "otherUsage");
75 }
apps/server/src/orchestration-v2/testkit/fixtures/claude_idle_resume/output.ts
+11
@@ -32,4 +32,15 @@ export function assertClaudeIdleResumeOutput(
32 "idle resume first turn complete",
33 "idle resume second turn complete",
34 ]);
35 + // The resumed process's ledger starts at zero, so the first turn is not counted again.
36 + assert.deepEqual(
37 + projection.providerTurns.map((turn) => [
38 + turn.turnTokenUsage?.outputTokens,
39 + turn.turnTokenUsage?.otherUsage,
40 + ]),
41 + [
42 + [7, { inputTokens: 0, outputTokens: 0 }],
43 + [7, { inputTokens: 0, outputTokens: 0 }],
44 + ],
45 + );
46 }
apps/server/src/orchestration-v2/testkit/fixtures/claude_nested_subagent_model/output.ts
+12
@@ -56,4 +56,16 @@ export function assertClaudeNestedSubagentModelOutput(
56 const nestedChild = result.projections.get(nested.childThreadId);
57 assert.isDefined(nestedChild);
58 assert.equal(nestedChild.thread.modelSelection.model, SUBAGENT_MODEL);
59 +
60 + // The result counts only the root; the process ledger adds both subagents' work once.
61 + assert.lengthOf(projection.providerTurns, 1);
62 + assert.deepInclude(projection.providerTurns[0]?.turnTokenUsage, {
63 + usageStatus: "complete",
64 + hasSubagents: true,
65 + inputTokens: 74_167,
66 + outputTokens: 236,
67 + reasoningTokens: 30,
68 + models: ["claude-sonnet-4-6"],
69 + otherUsage: { inputTokens: 87_409, outputTokens: 335 },
70 + });
71 }
apps/server/src/orchestration-v2/testkit/fixtures/claude_result_is_error/output.ts
+2
@@ -40,6 +40,8 @@ export function assertClaudeResultIsErrorOutput(
40 (turn) => turn.status !== "completed" && turn.status !== "running",
41 );
42 assert.equal(failedTurn?.status, "failed");
43 + // The API error's "<synthetic>" frame is a CLI notice, not a model that did work.
44 + assert.notProperty(failedTurn?.turnTokenUsage, "models");
45
46 const errorItem = projection.turnItems.find(
47 (item) => item.runId === failedRun?.id && item.type === "error",
apps/server/src/orchestration-v2/testkit/fixtures/codex_model_reroute/codex_transcript.ndjson new
+47
@@ -0,0 +1,47 @@
1 +{"type":"transcript_start","provider":"codex","protocol":"codex.app-server","version":"0.156.1","scenario":"codex_model_reroute","metadata":{"source":"record-codex-app-server-replay-fixture","fileName":"multi_turn.ndjson","description":"The multi_turn recording with a model/rerouted notification added by hand to the first turn. The second turn gets no new thread settings.","model":"gpt-6-luna"}}
2 +{"type":"expect_outbound","label":"initialize","frame":{"id":1,"method":"initialize","params":{"capabilities":{"experimentalApi":true,"optOutNotificationMethods":["turn/diff/updated"],"extensions":{"io.modelcontextprotocol/ui":{"mimeTypes":["text/html;profile=mcp-app"]}}},"clientInfo":{"name":"T3 Code","title":"T3 Code","version":"0.0.44"}}}}
3 +{"type":"emit_inbound","label":"initialize","frame":{"id":1,"result":{"userAgent":"t3code_desktop/0.156.1 (Debian 13.0.0; x86_64) unknown (t3code_desktop; 0.1.0)","codexHome":"/home/replay-user/.codex","platformFamily":"unix","platformOs":"linux"}}}
4 +{"type":"expect_outbound","label":"initialized","frame":{"method":"initialized"}}
5 +{"type":"expect_outbound","label":"thread/start","frame":{"id":2,"method":"thread/start","params":{"config":{"tools.update_plan.enabled":true,"mcp_optional_startup_grace_ms":10000},"cwd":"<workspace>","model":"gpt-6-luna"}}}
6 +{"type":"emit_inbound","label":"configWarning","frame":{"method":"configWarning","params":{"summary":"Codex could not find bubblewrap on PATH. Install bubblewrap with your OS package manager. See the sandbox prerequisites: https://developers.openai.com/codex/concepts/sandboxing#prerequisites. Codex will use the bundled bubblewrap in the meantime.","details":null},"emittedAtMs":1790295480414}}
7 +{"type":"emit_inbound","label":"remoteControl/status/changed","frame":{"method":"remoteControl/status/changed","params":{"status":"disabled","serverName":"replay-host","installationId":"00000000-0000-4000-8000-000000000000","environmentId":null},"emittedAtMs":1790295480414}}
8 +{"type":"emit_inbound","label":"thread/start","frame":{"id":2,"result":{"thread":{"id":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","environments":[{"environmentId":"local","cwd":"/home/replay-user/t3code/packages/effect-codex-app-server","runtimeWorkspaceRoots":["/home/replay-user/t3code/packages/effect-codex-app-server"]}],"extra":null,"sessionId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","forkedFromId":null,"parentThreadId":null,"preview":"","ephemeral":false,"section":null,"sectionEnteredAt":null,"projectId":null,"historyMode":"paginated","modelProvider":"openai","model":"gpt-6-luna","reasoningEffort":null,"createdAt":1790295480,"updatedAt":1790295480,"recencyAt":1790295480,"status":{"type":"idle"},"path":"/home/replay-user/.codex/sessions/2026/09/24/rollout-2026-09-24T17-18-00-01a0d5ed-187d-70c2-b4b6-b6881a68de09.jsonl","cwd":"/home/replay-user/t3code/packages/effect-codex-app-server","cliVersion":"0.156.1","originator":"t3code_desktop","source":"vscode","canAcceptDirectInput":true,"threadSource":null,"agentNickname":null,"agentRole":null,"gitInfo":null,"name":null,"daybreakEnabled":null,"turns":[]},"model":"gpt-6-luna","modelProvider":"openai","serviceTier":null,"disabledPluginIds":[],"cwd":"/home/replay-user/t3code/packages/effect-codex-app-server","runtimeWorkspaceRoots":["/home/replay-user/t3code/packages/effect-codex-app-server"],"instructionSources":["/home/replay-user/.codex/AGENTS.md","/home/replay-user/t3code/AGENTS.md"],"approvalPolicy":"on-request","approvalsReviewer":"user","sandbox":{"type":"workspaceWrite","writableRoots":[],"networkAccess":false,"excludeTmpdirEnvVar":false,"excludeSlashTmp":false},"activePermissionProfile":{"id":":workspace","extends":null},"reasoningEffort":null,"multiAgentMode":"explicitRequestOnly"}}}
9 +{"type":"expect_outbound","label":"turn/start","frame":{"id":3,"method":"turn/start","params":{"approvalPolicy":"never","approvalsReviewer":"user","cwd":"<workspace>","input":[{"text":"Respond with exactly: first fixture turn complete","type":"text"}],"model":"gpt-6-luna","sandboxPolicy":{"type":"dangerFullAccess"},"summary":"detailed","threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","collaborationMode":{"mode":"default","settings":{"model":"gpt-6-luna"}}}}}
10 +{"type":"emit_inbound","label":"thread/started","frame":{"method":"thread/started","params":{"thread":{"id":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","environments":[{"environmentId":"local","cwd":"/home/replay-user/t3code/packages/effect-codex-app-server","runtimeWorkspaceRoots":["/home/replay-user/t3code/packages/effect-codex-app-server"]}],"extra":null,"sessionId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","forkedFromId":null,"parentThreadId":null,"preview":"","ephemeral":false,"section":null,"sectionEnteredAt":null,"projectId":null,"historyMode":"paginated","modelProvider":"openai","model":"gpt-6-luna","reasoningEffort":null,"createdAt":1790295480,"updatedAt":1790295480,"recencyAt":1790295480,"status":{"type":"idle"},"path":"/home/replay-user/.codex/sessions/2026/09/24/rollout-2026-09-24T17-18-00-01a0d5ed-187d-70c2-b4b6-b6881a68de09.jsonl","cwd":"/home/replay-user/t3code/packages/effect-codex-app-server","cliVersion":"0.156.1","originator":"t3code_desktop","source":"vscode","canAcceptDirectInput":true,"threadSource":null,"agentNickname":null,"agentRole":null,"gitInfo":null,"name":null,"daybreakEnabled":null,"turns":[]}},"emittedAtMs":1790295480656}}
11 +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","name":"codex_apps","status":"starting","error":null,"failureReason":null},"emittedAtMs":1790295480656}}
12 +{"type":"emit_inbound","label":"thread/settings/updated","frame":{"method":"thread/settings/updated","params":{"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","threadSettings":{"disabledPluginIds":[],"cwd":"/home/replay-user/t3code/packages/effect-codex-app-server","approvalPolicy":"never","approvalsReviewer":"user","sandboxPolicy":{"type":"dangerFullAccess"},"activePermissionProfile":null,"model":"gpt-6-luna","modelProvider":"openai","serviceTier":null,"effort":null,"summary":"detailed","collaborationMode":{"mode":"default","settings":{"model":"gpt-6-luna","reasoning_effort":null,"developer_instructions":null}},"multiAgentMode":"explicitRequestOnly","personality":null}},"emittedAtMs":1790295480666}}
13 +{"type":"emit_inbound","label":"turn/start","frame":{"id":3,"result":{"turn":{"id":"01a0d5ed-1957-7a22-8399-d0172af342fd","items":[],"itemsView":"notLoaded","status":"inProgress","error":null,"startedAt":null,"completedAt":null,"durationMs":null}}}}
14 +{"type":"emit_inbound","label":"thread/status/changed","frame":{"method":"thread/status/changed","params":{"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","status":{"type":"active","activeFlags":[]}},"emittedAtMs":1790295480718}}
15 +{"type":"emit_inbound","label":"turn/started","frame":{"method":"turn/started","params":{"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","turn":{"id":"01a0d5ed-1957-7a22-8399-d0172af342fd","items":[],"itemsView":"notLoaded","status":"inProgress","error":null,"startedAt":1790295480,"completedAt":null,"durationMs":null}},"emittedAtMs":1790295480718}}
16 +{"type":"emit_inbound","label":"model/rerouted","frame":{"method":"model/rerouted","params":{"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","turnId":"01a0d5ed-1957-7a22-8399-d0172af342fd","fromModel":"gpt-6-luna","toModel":"gpt-6-astra","reason":"highRiskCyberActivity"}}}
17 +{"type":"emit_inbound","label":"account/updated","frame":{"method":"account/updated","params":{"authMode":"chatgpt","planType":"unknown"},"emittedAtMs":1790295481015}}
18 +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","name":"xcodebuildmcp","status":"starting","error":null,"failureReason":null},"emittedAtMs":1790295481236}}
19 +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","name":"codex_apps","status":"ready","error":null,"failureReason":null},"emittedAtMs":1790295481609}}
20 +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"userMessage","id":"01a0d5ed-22a8-7e91-9510-1578c7566bc0","clientId":null,"content":[{"type":"text","text":"Respond with exactly: first fixture turn complete","text_elements":[]}]},"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","turnId":"01a0d5ed-1957-7a22-8399-d0172af342fd","startedAtMs":1790295483054},"emittedAtMs":1790295483054}}
21 +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"userMessage","id":"01a0d5ed-22a8-7e91-9510-1578c7566bc0","clientId":null,"content":[{"type":"text","text":"Respond with exactly: first fixture turn complete","text_elements":[]}]},"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","turnId":"01a0d5ed-1957-7a22-8399-d0172af342fd","completedAtMs":1790295483054},"emittedAtMs":1790295483061}}
22 +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"agentMessage","id":"msg_06900ecb02a780a6016ab5bdbcd76887d0ba52ca8fee9da2e1","text":"","phase":"final_answer","memoryCitation":null,"delivery":null,"questions":null},"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","turnId":"01a0d5ed-1957-7a22-8399-d0172af342fd","startedAtMs":1790295485125},"emittedAtMs":1790295485125}}
23 +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","turnId":"01a0d5ed-1957-7a22-8399-d0172af342fd","itemId":"msg_06900ecb02a780a6016ab5bdbcd76887d0ba52ca8fee9da2e1","delta":"first"},"emittedAtMs":1790295485125}}
24 +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","turnId":"01a0d5ed-1957-7a22-8399-d0172af342fd","itemId":"msg_06900ecb02a780a6016ab5bdbcd76887d0ba52ca8fee9da2e1","delta":" fixture"},"emittedAtMs":1790295485125}}
25 +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","turnId":"01a0d5ed-1957-7a22-8399-d0172af342fd","itemId":"msg_06900ecb02a780a6016ab5bdbcd76887d0ba52ca8fee9da2e1","delta":" turn"},"emittedAtMs":1790295485125}}
26 +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","turnId":"01a0d5ed-1957-7a22-8399-d0172af342fd","itemId":"msg_06900ecb02a780a6016ab5bdbcd76887d0ba52ca8fee9da2e1","delta":" complete"},"emittedAtMs":1790295485125}}
27 +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"agentMessage","id":"msg_06900ecb02a780a6016ab5bdbcd76887d0ba52ca8fee9da2e1","text":"first fixture turn complete","phase":"final_answer","memoryCitation":null,"delivery":null,"questions":null},"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","turnId":"01a0d5ed-1957-7a22-8399-d0172af342fd","completedAtMs":1790295485125},"emittedAtMs":1790295485134}}
28 +{"type":"emit_inbound","label":"thread/tokenUsage/updated","frame":{"method":"thread/tokenUsage/updated","params":{"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","turnId":"01a0d5ed-1957-7a22-8399-d0172af342fd","tokenUsage":{"total":{"totalTokens":21426,"inputTokens":21418,"cachedInputTokens":6912,"cacheWriteInputTokens":0,"outputTokens":8,"reasoningOutputTokens":0},"last":{"totalTokens":21426,"inputTokens":21418,"cachedInputTokens":6912,"cacheWriteInputTokens":0,"outputTokens":8,"reasoningOutputTokens":0},"modelContextWindow":258400}},"emittedAtMs":1790295485211}}
29 +{"type":"emit_inbound","label":"thread/status/changed","frame":{"method":"thread/status/changed","params":{"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","status":{"type":"idle"}},"emittedAtMs":1790295485220}}
30 +{"type":"emit_inbound","label":"turn/completed","frame":{"method":"turn/completed","params":{"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","turn":{"id":"01a0d5ed-1957-7a22-8399-d0172af342fd","items":[{"type":"agentMessage","id":"msg_06900ecb02a780a6016ab5bdbcd76887d0ba52ca8fee9da2e1","text":"first fixture turn complete","phase":"final_answer","memoryCitation":null,"delivery":null,"questions":null}],"itemsView":"summary","status":"completed","error":null,"startedAt":1790295480,"completedAt":1790295485,"durationMs":4550}},"emittedAtMs":1790295485220}}
31 +{"type":"expect_outbound","label":"turn/start","frame":{"id":4,"method":"turn/start","params":{"approvalPolicy":"never","approvalsReviewer":"user","cwd":"<workspace>","input":[{"text":"Respond with exactly: second fixture turn complete","type":"text"}],"model":"gpt-6-luna","sandboxPolicy":{"type":"dangerFullAccess"},"summary":"detailed","threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","collaborationMode":{"mode":"default","settings":{"model":"gpt-6-luna"}}}}}
32 +{"type":"emit_inbound","label":"mcpServer/startupStatus/updated","frame":{"method":"mcpServer/startupStatus/updated","params":{"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","name":"xcodebuildmcp","status":"ready","error":null,"failureReason":null},"emittedAtMs":1790295485265}}
33 +{"type":"emit_inbound","label":"turn/start","frame":{"id":4,"result":{"turn":{"id":"01a0d5ed-2b29-7353-a6d5-7745731173b7","items":[],"itemsView":"notLoaded","status":"inProgress","error":null,"startedAt":null,"completedAt":null,"durationMs":null}}}}
34 +{"type":"emit_inbound","label":"thread/status/changed","frame":{"method":"thread/status/changed","params":{"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","status":{"type":"active","activeFlags":[]}},"emittedAtMs":1790295485286}}
35 +{"type":"emit_inbound","label":"turn/started","frame":{"method":"turn/started","params":{"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","turn":{"id":"01a0d5ed-2b29-7353-a6d5-7745731173b7","items":[],"itemsView":"notLoaded","status":"inProgress","error":null,"startedAt":1790295485,"completedAt":null,"durationMs":null}},"emittedAtMs":1790295485286}}
36 +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"userMessage","id":"01a0d5ed-2bdf-7093-9e82-b961af1fe132","clientId":null,"content":[{"type":"text","text":"Respond with exactly: second fixture turn complete","text_elements":[]}]},"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","turnId":"01a0d5ed-2b29-7353-a6d5-7745731173b7","startedAtMs":1790295485409},"emittedAtMs":1790295485409}}
37 +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"userMessage","id":"01a0d5ed-2bdf-7093-9e82-b961af1fe132","clientId":null,"content":[{"type":"text","text":"Respond with exactly: second fixture turn complete","text_elements":[]}]},"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","turnId":"01a0d5ed-2b29-7353-a6d5-7745731173b7","completedAtMs":1790295485409},"emittedAtMs":1790295485413}}
38 +{"type":"emit_inbound","label":"item/started","frame":{"method":"item/started","params":{"item":{"type":"agentMessage","id":"msg_06900ecb02a780a6016ab5bdbf08a087d08c1dafcd3f15697a","text":"","phase":"final_answer","memoryCitation":null,"delivery":null,"questions":null},"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","turnId":"01a0d5ed-2b29-7353-a6d5-7745731173b7","startedAtMs":1790295487332},"emittedAtMs":1790295487332}}
39 +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","turnId":"01a0d5ed-2b29-7353-a6d5-7745731173b7","itemId":"msg_06900ecb02a780a6016ab5bdbf08a087d08c1dafcd3f15697a","delta":"second"},"emittedAtMs":1790295487332}}
40 +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","turnId":"01a0d5ed-2b29-7353-a6d5-7745731173b7","itemId":"msg_06900ecb02a780a6016ab5bdbf08a087d08c1dafcd3f15697a","delta":" fixture"},"emittedAtMs":1790295487332}}
41 +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","turnId":"01a0d5ed-2b29-7353-a6d5-7745731173b7","itemId":"msg_06900ecb02a780a6016ab5bdbf08a087d08c1dafcd3f15697a","delta":" turn"},"emittedAtMs":1790295487332}}
42 +{"type":"emit_inbound","label":"item/agentMessage/delta","frame":{"method":"item/agentMessage/delta","params":{"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","turnId":"01a0d5ed-2b29-7353-a6d5-7745731173b7","itemId":"msg_06900ecb02a780a6016ab5bdbf08a087d08c1dafcd3f15697a","delta":" complete"},"emittedAtMs":1790295487332}}
43 +{"type":"emit_inbound","label":"item/completed","frame":{"method":"item/completed","params":{"item":{"type":"agentMessage","id":"msg_06900ecb02a780a6016ab5bdbf08a087d08c1dafcd3f15697a","text":"second fixture turn complete","phase":"final_answer","memoryCitation":null,"delivery":null,"questions":null},"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","turnId":"01a0d5ed-2b29-7353-a6d5-7745731173b7","completedAtMs":1790295487382},"emittedAtMs":1790295487398}}
44 +{"type":"emit_inbound","label":"thread/tokenUsage/updated","frame":{"method":"thread/tokenUsage/updated","params":{"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","turnId":"01a0d5ed-2b29-7353-a6d5-7745731173b7","tokenUsage":{"total":{"totalTokens":47211,"inputTokens":47195,"cachedInputTokens":28160,"cacheWriteInputTokens":0,"outputTokens":16,"reasoningOutputTokens":0},"last":{"totalTokens":25785,"inputTokens":25777,"cachedInputTokens":21248,"cacheWriteInputTokens":0,"outputTokens":8,"reasoningOutputTokens":0},"modelContextWindow":258400}},"emittedAtMs":1790295487407}}
45 +{"type":"emit_inbound","label":"thread/status/changed","frame":{"method":"thread/status/changed","params":{"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","status":{"type":"idle"}},"emittedAtMs":1790295487412}}
46 +{"type":"emit_inbound","label":"turn/completed","frame":{"method":"turn/completed","params":{"threadId":"01a0d5ed-187d-70c2-b4b6-b6881a68de09","turn":{"id":"01a0d5ed-2b29-7353-a6d5-7745731173b7","items":[{"type":"agentMessage","id":"msg_06900ecb02a780a6016ab5bdbf08a087d08c1dafcd3f15697a","text":"second fixture turn complete","phase":"final_answer","memoryCitation":null,"delivery":null,"questions":null}],"itemsView":"summary","status":"completed","error":null,"startedAt":1790295485,"completedAt":1790295487,"durationMs":2138}},"emittedAtMs":1790295487412}}
47 +{"type":"runtime_exit","status":"success"}
apps/server/src/orchestration-v2/testkit/fixtures/codex_model_reroute/output.ts new
+21
@@ -0,0 +1,21 @@
1 +import { assert } from "@effect/vitest";
2 +import type { ProviderReplayTranscript } from "@t3tools/contracts";
3 +
4 +import type { OrchestratorV2ScenarioResult } from "../../OrchestratorScenario.ts";
5 +import { assertBaseProjection, projectionFor } from "../shared.ts";
6 +
7 +export function assertCodexModelRerouteOutput(
8 + result: OrchestratorV2ScenarioResult,
9 + transcript: ProviderReplayTranscript,
10 +) {
11 + assertBaseProjection({
12 + result,
13 + transcript,
14 + runCount: 2,
15 + runStatuses: ["completed", "completed"],
16 + });
17 + const [first, second] = projectionFor(result, transcript.scenario).providerTurns;
18 + // The reroute names the first turn only; without new settings the second turn's model is unknown.
19 + assert.deepEqual(first?.turnTokenUsage?.models, ["gpt-6-astra"]);
20 + assert.notProperty(second?.turnTokenUsage, "models");
21 +}
apps/server/src/orchestration-v2/testkit/fixtures/index.ts
+29
@@ -106,6 +106,8 @@ import { openCode2SimpleInput } from "./opencode2_simple/input.ts";
106 import { openCode2SkillInput } from "./opencode2_skill/input.ts";
107 import { assertOpenCode2SkillOutput } from "./opencode2_skill/output.ts";
108 import { assertOpenCode2SimpleOutput } from "./opencode2_simple/output.ts";
109 +import { assertOpenCode2MissingStepUsageOutput } from "./opencode2_missing_step_usage/output.ts";
110 +import { assertCodexModelRerouteOutput } from "./codex_model_reroute/output.ts";
111 import { openCode2ToolCallInput } from "./opencode2_tool_call/input.ts";
112 import { assertOpenCode2ToolCallOutput } from "./opencode2_tool_call/output.ts";
113 import { assertOpenCodeSubagentOutput } from "./opencode_subagent/output.ts";
@@ -984,6 +986,21 @@ export const ORCHESTRATOR_REPLAY_FIXTURES: ReadonlyArray<OrchestratorReplayFixtu
986 },
987 ],
988 },
989 + {
990 + name: "opencode2_missing_step_usage",
991 + buildInput: openCode2SimpleInput,
992 + providers: [
993 + {
994 + driver: ProviderDriverKind.make("opencode"),
995 + transcriptFile: new URL(
996 + "./opencode2_missing_step_usage/opencode_transcript.ndjson",
997 + import.meta.url,
998 + ),
999 + modelSelection: OPENCODE2_MODEL_SELECTION,
1000 + assertOutput: assertOpenCode2MissingStepUsageOutput,
1001 + },
1002 + ],
1003 + },
1004 {
1005 name: "opencode2_tool_call",
1006 buildInput: openCode2ToolCallInput,
@@ -1221,6 +1238,18 @@ export const ORCHESTRATOR_REPLAY_FIXTURES: ReadonlyArray<OrchestratorReplayFixtu
1238 },
1239 ],
1240 },
1241 + {
1242 + name: "codex_model_reroute",
1243 + buildInput: multiTurnInput,
1244 + providers: [
1245 + {
1246 + driver: ProviderDriverKind.make("codex"),
1247 + transcriptFile: new URL("./codex_model_reroute/codex_transcript.ndjson", import.meta.url),
1248 + modelSelection: CODEX_MODEL_SELECTION,
1249 + assertOutput: assertCodexModelRerouteOutput,
1250 + },
1251 + ],
1252 + },
1253 {
1254 name: "multi_turn",
1255 buildInput: multiTurnInput,
apps/server/src/orchestration-v2/testkit/fixtures/opencode2_missing_step_usage/opencode_transcript.ndjson new
+55
@@ -0,0 +1,55 @@
1 +{"type": "transcript_start", "provider": "opencode", "protocol": "opencode2-http.sse", "version": "2.0.18", "scenario": "opencode2_missing_step_usage", "metadata": {"source": "opencode2-spike/recordings/simple.ndjson", "capturedAt": "2026-09-28", "nativeSessionId": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "description": "The opencode2_simple recording with a second step added by hand that fails without a token report, so the turn has one measured step and one missing report."}}
2 +{"type": "expect_outbound", "label": "event.subscribe", "frame": {"type": "event.subscribe"}}
3 +{"type": "expect_outbound", "label": "model.list", "frame": {"type": "model.list", "input": {"location[directory]": "<any>"}}}
4 +{"type": "emit_inbound", "label": "model.list.response", "frame": {"type": "sdk.response", "operation": "model.list", "data": {"location": {"directory": "<work>"}, "data": [{"id": "space-bunny-free", "modelID": "space-bunny-free", "providerID": "opencode", "name": "Space Bunny Free", "compatibility": {"reasoningField": "reasoning_content"}, "package": "@opencode/ai/providers/openai-compatible", "settings": {"apiKey": "public", "baseURL": "https://opencode.ai/zen/v1", "provider": "opencode"}, "capabilities": {"tools": true, "input": ["text", "image", "video"], "output": ["text"]}, "variants": [{"id": "low", "settings": {"reasoningEffort": "low"}}, {"id": "medium", "settings": {"reasoningEffort": "medium"}}, {"id": "high", "settings": {"reasoningEffort": "high"}}, {"id": "xhigh", "settings": {"reasoningEffort": "xhigh"}}, {"id": "max", "settings": {"reasoningEffort": "max"}}], "time": {"released": 1790121600000}, "cost": [{"input": 0, "output": 0, "cache": {"read": 0, "write": 0}}], "status": "active", "enabled": true, "limit": {"context": 1048576, "input": 524288, "output": 524288}}, {"id": "big-pickle", "modelID": "big-pickle", "providerID": "opencode", "family": "big-pickle", "name": "Big Pickle", "compatibility": {"reasoningField": "reasoning_content"}, "package": "@opencode/ai/providers/openai-compatible", "settings": {"apiKey": "public", "baseURL": "https://opencode.ai/zen/v1", "provider": "opencode"}, "capabilities": {"tools": true, "input": ["text"], "output": ["text"]}, "variants": [], "time": {"released": 1760659200000}, "cost": [{"input": 0, "output": 0, "cache": {"read": 0, "write": 0}}], "status": "active", "enabled": true, "limit": {"context": 200000, "input": 160000, "output": 32000}}]}}}
5 +{"type": "emit_inbound", "label": "server.connected", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb73205b001QM7spc7rTwRevR", "type": "server.connected", "data": {}}}}
6 +{"type": "expect_outbound", "label": "session.create", "frame": {"type": "session.create", "input": {"location": "<any>", "model": {"providerID": "opencode", "id": "space-bunny-free", "variant": "high"}, "permissions": "<any>"}}}
7 +{"type": "emit_inbound", "label": "session.create.response", "frame": {"type": "sdk.response", "operation": "session.create", "data": {"data": {"id": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "projectID": "global", "model": {"id": "space-bunny-free", "providerID": "opencode", "variant": "high"}, "cost": 0, "tokens": {"input": 0, "output": 0, "reasoning": 0, "cache": {"read": 0, "write": 0}}, "time": {"created": 1790656585840, "updated": 1790656585840}, "title": "spike simple", "location": {"directory": "<work>"}}}}}
8 +{"type": "emit_inbound", "label": "session.created", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb732070001MeC12AQ1Ds6a0a", "created": 1790656585840, "type": "session.created", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "slug": "sunny-pixel", "version": "2.0.18", "projectID": "global", "location": {"directory": "<work>"}, "subpath": "", "title": "spike simple", "model": {"id": "space-bunny-free", "providerID": "opencode", "variant": "high"}}, "durable": {"aggregateID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "seq": 0, "version": 1}}}}
9 +{"type": "emit_inbound", "label": "project.updated", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb732078001oO5ghtqXkMav9d", "created": 1790656585848, "type": "project.updated", "data": {"id": "global", "canonical": "<work>", "vcs": "git", "time": {"created": 1790656063718, "updated": 1790656063718, "active": 1790656585847}, "sandboxes": []}}}}
10 +{"type": "expect_outbound", "label": "session.instructions.entry.put", "frame": {"type": "session.instructions.entry.put", "input": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "key": "t3-code", "value": "<any>"}}}
11 +{"type": "emit_inbound", "label": "session.instructions.entry.put.response", "frame": {"type": "sdk.response", "operation": "session.instructions.entry.put", "data": null}}
12 +{"type": "expect_outbound", "label": "session.prompt", "frame": {"type": "session.prompt", "input": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "text": "<any>"}}}
13 +{"type": "emit_inbound", "label": "session.prompt.response", "frame": {"type": "sdk.response", "operation": "session.prompt", "data": {"data": {"id": "msg_0eb732081001RntUJfRtTXOAjd", "sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "time": {"created": 1790656585859}, "type": "user", "payload": {"text": "Think carefully step by step about whether 391 is prime, showing your reasoning, then answer in one short sentence."}, "delivery": "steer"}}}}
14 +{"type": "emit_inbound", "label": "session.inbox.enqueued", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb732083001UPoh8n1ciLGG4J", "created": 1790656585859, "type": "session.inbox.enqueued", "location": {"directory": "<work>"}, "data": {"inboxID": "msg_0eb732081001RntUJfRtTXOAjd", "sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "item": {"type": "user", "payload": {"text": "Think carefully step by step about whether 391 is prime, showing your reasoning, then answer in one short sentence."}, "delivery": "steer"}}, "durable": {"aggregateID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "seq": 1, "version": 1}}}}
15 +{"type": "emit_inbound", "label": "session.execution.started", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb732089001TRfMB4IJIUtJuG", "created": 1790656585865, "type": "session.execution.started", "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk"}, "durable": {"aggregateID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "seq": 2, "version": 1}}}}
16 +{"type": "emit_inbound", "label": "session.instructions.updated", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb732098001v3n3L11aMx0r6G", "created": 1790656585880, "metadata": {"instructions": {"initial": true}}, "type": "session.instructions.updated", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "delta": {"core/environment": "27a84baf7fee37fde05d49a6aaf0f184019068a15362a38f5d1ec17ac7e8f2b7", "core/date": "bf08b8486572ab3eef677a3c9ee12468e411ae41a8f4d2fb6cf02a6ca73f1986", "core/codemode": "616a7ad8b8e40e8a71eab238dfbbe9eaf0f4a9e6fdad853226d075bfe602e1e6", "core/skill-guidance": "0d001919496102654717308c72635523bf7c3cd99c81e058cc79f2947104f9a5"}}, "durable": {"aggregateID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "seq": 3, "version": 2}}}}
17 +{"type": "emit_inbound", "label": "session.inbox.delivered", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb73209d001wgPxWUQRwkXvBn", "created": 1790656585885, "type": "session.inbox.delivered", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "inboxID": "msg_0eb732081001RntUJfRtTXOAjd"}, "durable": {"aggregateID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "seq": 4, "version": 1}}}}
18 +{"type": "emit_inbound", "label": "session.step.started", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb7326930015SMmHkSszcae37", "created": 1790656587411, "type": "session.step.started", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "agent": "build", "model": {"id": "space-bunny-free", "providerID": "opencode", "variant": "high"}, "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "snapshot": "99482b54df6361365fa0ed7363ccf07edaa7fc93", "started": 1790656585925}, "durable": {"aggregateID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "seq": 5, "version": 1}}}}
19 +{"type": "emit_inbound", "label": "session.reasoning.started", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb732699001FC5efnDXzkrBV3", "created": 1790656587417, "type": "session.reasoning.started", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "ordinal": 0, "state": {"reasoningField": "reasoning_content"}}, "durable": {"aggregateID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "seq": 6, "version": 1}}}}
20 +{"type": "emit_inbound", "label": "session.reasoning.delta", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb732702001XkfhcVQWas7guy", "created": 1790656587522, "type": "session.reasoning.delta", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "ordinal": 0, "delta": "Check divisibility up to sqrt("}}}}
21 +{"type": "emit_inbound", "label": "session.reasoning.delta.2", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb73277b001U6tDaZPzh5G8Ub", "created": 1790656587643, "type": "session.reasoning.delta", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "ordinal": 0, "delta": "391) \u2248 19.77."}}}}
22 +{"type": "emit_inbound", "label": "session.reasoning.delta.3", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb732805001zRua1yhKskE6du", "created": 1790656587781, "type": "session.reasoning.delta", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "ordinal": 0, "delta": " Primes: 2,3,5,7,"}}}}
23 +{"type": "emit_inbound", "label": "session.reasoning.delta.4", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb73286e001XXnCtQeutby7Jh", "created": 1790656587886, "type": "session.reasoning.delta", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "ordinal": 0, "delta": "11,13,17,19.\n\n391 is"}}}}
24 +{"type": "emit_inbound", "label": "session.reasoning.delta.5", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb7328e8001vpkz0eDsHflXmv", "created": 1790656588008, "type": "session.reasoning.delta", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "ordinal": 0, "delta": " odd, not div by 2"}}}}
25 +{"type": "emit_inbound", "label": "session.reasoning.delta.6", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb732955001YnC9O9FAgIagoL", "created": 1790656588118, "type": "session.reasoning.delta", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "ordinal": 0, "delta": ". 3+9+"}}}}
26 +{"type": "emit_inbound", "label": "session.reasoning.delta.7", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb732aaa001b129VgQ4aT7avM", "created": 1790656588458, "type": "session.reasoning.delta", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "ordinal": 0, "delta": "1=13, not div by 3. Not by 5. 7: 7\u00d755=385, remainder 6. 11: 11\u00d7"}}}}
27 +{"type": "emit_inbound", "label": "session.reasoning.delta.8", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb732b29001AMm839z4dDYXHE", "created": 1790656588585, "type": "session.reasoning.delta", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "ordinal": 0, "delta": "35=385, remainder 6. 13: "}}}}
28 +{"type": "emit_inbound", "label": "session.reasoning.delta.9", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb732ba0001Ivdi55eX7RWaIN", "created": 1790656588704, "type": "session.reasoning.delta", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "ordinal": 0, "delta": "13\u00d730=390, remainder 1. "}}}}
29 +{"type": "emit_inbound", "label": "session.reasoning.delta.10", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb732c1f001l5voAVE8zZty1G", "created": 1790656588832, "type": "session.reasoning.delta", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "ordinal": 0, "delta": "17: 17\u00d723=391!"}}}}
30 +{"type": "emit_inbound", "label": "session.reasoning.delta.11", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb732c95001xtU1t86iQ32FcX", "created": 1790656588949, "type": "session.reasoning.delta", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "ordinal": 0, "delta": " 17\u00d723 = 391. Yes"}}}}
31 +{"type": "emit_inbound", "label": "session.reasoning.delta.12", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb732d3b001F922FhdbWHFFh9", "created": 1790656589115, "type": "session.reasoning.delta", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "ordinal": 0, "delta": ": 17\u00d723 = 340 +"}}}}
32 +{"type": "emit_inbound", "label": "session.reasoning.delta.13", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb732db1001Llbce6UsLrsMtr", "created": 1790656589234, "type": "session.reasoning.delta", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "ordinal": 0, "delta": " 51 = 391. So "}}}}
33 +{"type": "emit_inbound", "label": "session.reasoning.delta.14", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb732de6001fDagd9vuCaT77g", "created": 1790656589286, "type": "session.reasoning.delta", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "ordinal": 0, "delta": "391 is composite."}}}}
34 +{"type": "emit_inbound", "label": "session.text.started", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb732de7001zYvkQcsI5WOzgG", "created": 1790656589287, "type": "session.text.started", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "ordinal": 0}, "durable": {"aggregateID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "seq": 7, "version": 1}}}}
35 +{"type": "emit_inbound", "label": "session.text.delta", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb732e50001GdabW0WKYVl32T", "created": 1790656589392, "type": "session.text.delta", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "ordinal": 0, "delta": "\u221a391 \u2248 19.8,"}}}}
36 +{"type": "emit_inbound", "label": "session.text.delta.2", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb732ed2001IBPZhAC1kx4d1v", "created": 1790656589522, "type": "session.text.delta", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "ordinal": 0, "delta": " so I need to test primes "}}}}
37 +{"type": "emit_inbound", "label": "session.text.delta.3", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb732f38001E0KjfQvDXHZ7h1", "created": 1790656589624, "type": "session.text.delta", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "ordinal": 0, "delta": "2, 3, 5, 7, "}}}}
38 +{"type": "emit_inbound", "label": "session.text.delta.4", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb732fad001JzcESOkjYwU5ik", "created": 1790656589741, "type": "session.text.delta", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "ordinal": 0, "delta": "11, 13, 17, 19: it's"}}}}
39 +{"type": "emit_inbound", "label": "session.text.delta.5", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb73301c001UBnr5x154Roys2", "created": 1790656589852, "type": "session.text.delta", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "ordinal": 0, "delta": " odd, digits sum to 13, doesn't"}}}}
40 +{"type": "emit_inbound", "label": "session.text.delta.6", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb73308e001sYejhe1LAMtvcI", "created": 1790656589966, "type": "session.text.delta", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "ordinal": 0, "delta": " end in 0/5, and 7\u00d7"}}}}
41 +{"type": "emit_inbound", "label": "session.text.delta.7", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb7331030012XzmUwjcWGmPUe", "created": 1790656590084, "type": "session.text.delta", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "ordinal": 0, "delta": "55=385, 11"}}}}
42 +{"type": "emit_inbound", "label": "session.text.delta.8", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb7331700018ST8kVeG36WRUV", "created": 1790656590192, "type": "session.text.delta", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "ordinal": 0, "delta": "\u00d735=385, 13\u00d730=390 leave remain"}}}}
43 +{"type": "emit_inbound", "label": "session.text.delta.9", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb7331df001Nm2KLtcVET4KlU", "created": 1790656590304, "type": "session.text.delta", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "ordinal": 0, "delta": "ders \u2014 but 17\u00d723 = 391"}}}}
44 +{"type": "emit_inbound", "label": "session.text.delta.10", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb733251001HkPIEuVZxWlLIU", "created": 1790656590417, "type": "session.text.delta", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "ordinal": 0, "delta": " exactly, so the search"}}}}
45 +{"type": "emit_inbound", "label": "session.text.delta.11", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb7332bd001f1FZxccg5aNmhq", "created": 1790656590525, "type": "session.text.delta", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "ordinal": 0, "delta": " stops there.\n\n391 is not prime: it's the"}}}}
46 +{"type": "emit_inbound", "label": "session.text.delta.12", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb73333d001mIrQbkY6rbRMi3", "created": 1790656590654, "type": "session.text.delta", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "ordinal": 0, "delta": " product 17 \u00d7 23."}}}}
47 +{"type": "emit_inbound", "label": "session.reasoning.ended", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb7333720010sLSSNU5xRXUwa", "created": 1790656590707, "type": "session.reasoning.ended", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "ordinal": 0, "text": "Check divisibility up to sqrt(391) \u2248 19.77. Primes: 2,3,5,7,11,13,17,19.\n\n391 is odd, not div by 2. 3+9+1=13, not div by 3. Not by 5. 7: 7\u00d755=385, remainder 6. 11: 11\u00d735=385, remainder 6. 13: 13\u00d730=390, remainder 1. 17: 17\u00d723=391! 17\u00d723 = 391. Yes: 17\u00d723 = 340 + 51 = 391. So 391 is composite.", "state": {"reasoningField": "reasoning_content"}}, "durable": {"aggregateID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "seq": 8, "version": 1}}}}
48 +{"type": "emit_inbound", "label": "session.text.ended", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb733378001jAPLwPeycQ4M5d", "created": 1790656590712, "type": "session.text.ended", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "ordinal": 0, "text": "\u221a391 \u2248 19.8, so I need to test primes 2, 3, 5, 7, 11, 13, 17, 19: it's odd, digits sum to 13, doesn't end in 0/5, and 7\u00d755=385, 11\u00d735=385, 13\u00d730=390 leave remainders \u2014 but 17\u00d723 = 391 exactly, so the search stops there.\n\n391 is not prime: it's the product 17 \u00d7 23."}, "durable": {"aggregateID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "seq": 9, "version": 1}}}}
49 +{"type": "emit_inbound", "label": "session.step.streamed", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb73337f001Y6nJkjv1tXkqGt", "created": 1790656590719, "type": "session.step.streamed", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT"}, "durable": {"aggregateID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "seq": 10, "version": 1}}}}
50 +{"type": "emit_inbound", "label": "session.step.ended", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb733390001vg5GRCuNzMFbCA", "created": 1790656590736, "type": "session.step.ended", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7320a9001vve3OV5uNi2HRT", "finish": "stop", "rawFinish": "stop", "cost": 0, "tokens": {"input": 8701, "output": 113, "reasoning": 147, "cache": {"read": 489, "write": 0}}, "snapshot": "99482b54df6361365fa0ed7363ccf07edaa7fc93", "files": []}, "durable": {"aggregateID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "seq": 11, "version": 1}}}}
51 +{"type": "emit_inbound", "label": "session.usage.updated", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb733396001Ke5QhqiR5CdvP6", "created": 1790656590742, "type": "session.usage.updated", "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "cost": 0, "tokens": {"input": 8701, "output": 113, "reasoning": 147, "cache": {"read": 489, "write": 0}}}}}}
52 +{"type": "emit_inbound", "label": "session.step.started.2", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb7333a1001missingStepStart", "created": 1790656590750, "type": "session.step.started", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "agent": "build", "model": {"id": "space-bunny-free", "providerID": "opencode", "variant": "high"}, "assistantMessageID": "msg_0eb7333a0001missingStepUsage", "snapshot": "99482b54df6361365fa0ed7363ccf07edaa7fc93", "started": 1790656590750}, "durable": {"aggregateID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "seq": 12, "version": 1}}}}
53 +{"type": "emit_inbound", "label": "session.step.failed", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb7333a2001missingStepFailed", "created": 1790656590760, "type": "session.step.failed", "location": {"directory": "<work>"}, "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "assistantMessageID": "msg_0eb7333a0001missingStepUsage", "error": {"type": "provider_error", "message": "Stream closed before usage was reported"}}, "durable": {"aggregateID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "seq": 13, "version": 1}}}}
54 +{"type": "emit_inbound", "label": "session.execution.succeeded", "frame": {"type": "sdk.event", "event": {"id": "evt_0eb733399001NhwTrB32UU6d6H", "created": 1790656590745, "type": "session.execution.succeeded", "data": {"sessionID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk"}, "durable": {"aggregateID": "ses_f148cdfa1ffeIFwbC0FCx0fKpk", "seq": 14, "version": 1}}}}
55 +{"type": "runtime_exit", "status": "success"}
apps/server/src/orchestration-v2/testkit/fixtures/opencode2_missing_step_usage/output.ts new
+22
@@ -0,0 +1,22 @@
1 +import { assert } from "@effect/vitest";
2 +import type { ProviderReplayTranscript } from "@t3tools/contracts";
3 +
4 +import type { OrchestratorV2ScenarioResult } from "../../OrchestratorScenario.ts";
5 +import { assertBaseProjection, projectionFor } from "../shared.ts";
6 +
7 +export function assertOpenCode2MissingStepUsageOutput(
8 + result: OrchestratorV2ScenarioResult,
9 + transcript: ProviderReplayTranscript,
10 +) {
11 + assertBaseProjection({ result, transcript, runCount: 1, runStatuses: ["completed"] });
12 + const [providerTurn] = projectionFor(result, transcript.scenario).providerTurns;
13 + assert.equal(providerTurn?.status, "completed");
14 + // The measured step survives; the step without a report keeps the turn total partial.
15 + assert.deepInclude(providerTurn?.turnTokenUsage, {
16 + usageStatus: "partial",
17 + inputTokens: 8701 + 489,
18 + outputTokens: 113 + 147,
19 + reasoningTokens: 147,
20 + models: ["opencode/space-bunny-free"],
21 + });
22 +}
apps/server/src/orchestration-v2/testkit/fixtures/opencode2_simple/output.ts
+1
@@ -41,6 +41,7 @@ export function assertOpenCode2SimpleOutput(
41 cachedInputTokens: 489,
42 outputTokens: 113 + 147,
43 reasoningTokens: 147,
44 + models: ["opencode/space-bunny-free"],
45 });
46 assert.deepInclude(providerTurn?.tokenUsage, {
47 usedTokens: 8701 + 489 + 113,
apps/server/src/orchestration-v2/testkit/fixtures/subagent_v2/codex_output.ts
+10
@@ -83,4 +83,14 @@ export function assertSubagentV2Output(
83 "child thread must contain the post-root subagent response",
84 );
85 assert.equal(childProjection.providerTurns[0]?.status, "completed");
86 +
87 + // Codex reports each agent's counts on its own turn. Only the root's model was reported.
88 + assert.deepInclude(projection.providerTurns[0]?.turnTokenUsage, {
89 + usageStatus: "complete",
90 + outputTokens: 69,
91 + models: ["gpt-6-luna"],
92 + });
93 + const childUsage = childProjection.providerTurns[0]?.turnTokenUsage;
94 + assert.deepInclude(childUsage, { usageStatus: "complete", outputTokens: 6 });
95 + assert.notProperty(childUsage, "models");
96 }
apps/server/src/provider/ClaudeTurnTokenUsage.ts
+100 -1
@@ -49,12 +49,96 @@ function withEarlierSegmentUsage(
49 return total;
50 }
51
52 +/** Per-model totals from the SDK's `modelUsage`, which grows for the life of one CLI process. */
53 +export type ClaudeModelLedger = ReadonlyMap<
54 + string,
55 + { readonly input: number; readonly output: number }
56 +>;
57 +
58 +export interface ClaudeLedgerUsage {
59 + readonly inputTokens: number;
60 + readonly outputTokens: number;
61 +}
62 +
63 +/** Unknown when absent or empty: a process that did model work never reports an empty ledger. */
64 +export function claudeModelLedger(modelUsage: unknown): ClaudeModelLedger | undefined {
65 + if (typeof modelUsage !== "object" || modelUsage === null) return undefined;
66 + if (Object.keys(modelUsage).length === 0) return undefined;
67 + const ledger = new Map<string, { readonly input: number; readonly output: number }>();
68 + for (const [model, entry] of Object.entries(modelUsage as Record<string, unknown>)) {
69 + const fields = (entry ?? {}) as Record<string, unknown>;
70 + const input = finiteNonNegativeInteger(fields.inputTokens);
71 + const output = finiteNonNegativeInteger(fields.outputTokens);
72 + if (input === undefined || output === undefined) return undefined;
73 + const cacheRead = finiteNonNegativeInteger(fields.cacheReadInputTokens) ?? 0;
74 + const cacheWrite = finiteNonNegativeInteger(fields.cacheCreationInputTokens) ?? 0;
75 + ledger.set(model, { input: input + cacheRead + cacheWrite, output });
76 + }
77 + return ledger;
78 +}
79 +
80 +/** Ledger growth since `previous`. Unknown without a baseline or when a counter goes backwards. */
81 +export function claudeLedgerDelta(
82 + previous: ClaudeModelLedger | undefined,
83 + current: ClaudeModelLedger | undefined,
84 +): ClaudeLedgerUsage | undefined {
85 + if (previous === undefined || current === undefined) return undefined;
86 + let inputTokens = 0;
87 + let outputTokens = 0;
88 + for (const [model, before] of previous) {
89 + const after = current.get(model);
90 + if (after === undefined || after.input < before.input || after.output < before.output) {
91 + return undefined;
92 + }
93 + }
94 + for (const [model, after] of current) {
95 + const before = previous.get(model);
96 + inputTokens += after.input - (before?.input ?? 0);
97 + outputTokens += after.output - (before?.output ?? 0);
98 + }
99 + return { inputTokens, outputTokens };
100 +}
101 +
102 +export function addClaudeLedgerUsage(
103 + left: ClaudeLedgerUsage | undefined,
104 + right: ClaudeLedgerUsage | undefined,
105 +): ClaudeLedgerUsage | undefined {
106 + return left === undefined || right === undefined
107 + ? undefined
108 + : {
109 + inputTokens: left.inputTokens + right.inputTokens,
110 + outputTokens: left.outputTokens + right.outputTokens,
111 + };
112 +}
113 +
114 +export interface ClaudeTurnObservations {
115 + // Models on the turn's root assistant messages.
116 + readonly rootModels?: ReadonlySet<string>;
117 + // Ledger growth across every result this turn consumed, including steered segments.
118 + readonly ledger?: ClaudeLedgerUsage | undefined;
119 +}
120 +
121 export function normalizeClaudeTurnTokenUsage(
53 - result: { readonly subtype: SDKResultMessage["subtype"]; readonly usage?: unknown } | undefined,
122 + result:
123 + | {
124 + readonly subtype: SDKResultMessage["subtype"];
125 + readonly usage?: unknown;
126 + readonly local_command?: unknown;
127 + }
128 + | undefined,
129 hasSubagents: boolean,
130 terminalStatus: RuntimeTurnState,
131 earlierSegmentUsage: ReadonlyArray<unknown> = [],
132 + observed: ClaudeTurnObservations = {},
133 ): TurnTokenUsage {
134 + const observedFields = {
135 + ...(observed.rootModels !== undefined && observed.rootModels.size > 0
136 + ? { models: [...observed.rootModels] }
137 + : {}),
138 + ...(typeof result?.local_command === "string" && result.local_command.length > 0
139 + ? { localCommand: true }
140 + : {}),
141 + };
142 const finalUsage = result?.usage as Record<string, unknown> | undefined;
143 const usage = withEarlierSegmentUsage(finalUsage ?? {}, earlierSegmentUsage);
144 const uncachedInputTokens = finiteNonNegativeInteger(usage.input_tokens);
@@ -89,10 +173,25 @@ export function normalizeClaudeTurnTokenUsage(
173 usageStatus: "unavailable",
174 usageScope: "main_agent",
175 hasSubagents,
176 + ...observedFields,
177 };
178 }
179
180 + // The result omits subagent and compaction work that the process-wide ledger records.
181 + const otherUsage =
182 + observed.ledger !== undefined &&
183 + inputTokens !== undefined &&
184 + rawOutputTokens !== undefined &&
185 + observed.ledger.inputTokens >= inputTokens &&
186 + observed.ledger.outputTokens >= rawOutputTokens
187 + ? {
188 + inputTokens: observed.ledger.inputTokens - inputTokens,
189 + outputTokens: observed.ledger.outputTokens - rawOutputTokens,
190 + }
191 + : undefined;
192 const commonUsage = {
193 + ...observedFields,
194 + ...(otherUsage === undefined ? {} : { otherUsage }),
195 usageScope: "main_agent",
196 ...(cachedInputTokens !== undefined ? { cachedInputTokens } : {}),
197 ...(cacheCreationTokens !== undefined ? { cacheCreationTokens } : {}),
apps/server/src/provider/CodexTurnTokenUsage.ts
+7 -10
@@ -133,27 +133,23 @@ export function accumulateCodexTurnTokenUsage(
133 }
134 }
135
136 +/** `model` is the thread model Codex last reported, which a reroute replaces. */
137 export function completeCodexTurnTokenUsage(
138 state: CodexTurnTokenUsageState,
139 turnId: string,
140 completed: boolean,
141 + model?: string,
142 ): TurnTokenUsage {
143 const usage = state.byTurnId.get(turnId);
144 state.byTurnId.delete(turnId);
145 if (state.activeTurnId === turnId) state.activeTurnId = undefined;
144 - if (!usage) {
146 + const models = model === undefined ? {} : { models: [model] };
147 + if (!usage?.observed) {
148 return {
149 usageStatus: "unavailable",
150 usageScope: "main_agent",
148 - hasSubagents: false,
149 - };
150 - }
151 -
152 - if (!usage.observed) {
153 - return {
154 - usageStatus: "unavailable",
155 - usageScope: "main_agent",
156 - hasSubagents: usage.hasSubagents,
151 + hasSubagents: usage?.hasSubagents ?? false,
152 + ...models,
153 };
154 }
155
@@ -169,5 +165,6 @@ export function completeCodexTurnTokenUsage(
165 outputTokens: usage.outputTokens,
166 reasoningTokens: Math.min(usage.outputTokens, usage.reasoningTokens),
167 hasSubagents: usage.hasSubagents,
168 + ...models,
169 };
170 }
apps/server/src/provider/TurnTokenUsage.test.ts
+14 -1
@@ -1,5 +1,9 @@
1 import { describe, expect, it } from "vite-plus/test";
2 -import { normalizeClaudeTurnTokenUsage } from "./ClaudeTurnTokenUsage.ts";
2 +import {
3 + claudeLedgerDelta,
4 + claudeModelLedger,
5 + normalizeClaudeTurnTokenUsage,
6 +} from "./ClaudeTurnTokenUsage.ts";
7 import {
8 accumulateCodexTurnTokenUsage,
9 completeCodexTurnTokenUsage,
@@ -209,6 +213,15 @@ describe("billable native turn usage", () => {
213 },
214 );
215
216 + it("measures Claude ledger growth only from a known, nondecreasing baseline", () => {
217 + const ledger = (output: number) =>
218 + claudeModelLedger({ "claude-sonnet-4-6": { inputTokens: 100, outputTokens: output } });
219 + expect(claudeLedgerDelta(ledger(10), ledger(25))).toEqual({ inputTokens: 0, outputTokens: 15 });
220 + expect(claudeLedgerDelta(ledger(25), ledger(10))).toBeUndefined();
221 + expect(claudeLedgerDelta(undefined, ledger(10))).toBeUndefined();
222 + expect(claudeModelLedger({})).toBeUndefined();
223 + });
224 +
225 it("adds Claude steering segments to the final measured result once", () => {
226 expect(
227 normalizeClaudeTurnTokenUsage(
apps/server/src/server.ts
+2 -2
@@ -47,7 +47,7 @@ import * as PullRequestReviewGuideService from "./pullRequest/PullRequestReviewG
47 import * as ServerLifecycleEvents from "./serverLifecycleEvents.ts";
48 import * as AnalyticsService from "./telemetry/AnalyticsService.ts";
49 import * as DelegatedTaskAnalytics from "./telemetry/DelegatedTaskAnalytics.ts";
50 -import * as ProviderEventIngestor from "./orchestration-v2/ProviderEventIngestor.ts";
50 +import * as ProviderUsageAnalytics from "./telemetry/ProviderUsageAnalytics.ts";
51 import * as ModelManifest from "./provider/ModelManifest.ts";
52 import * as ResetCreditCoordinator from "./provider/resetCreditCoordinator.ts";
53 import * as EventNdjsonLogger from "./provider/EventNdjsonLogger.ts";
@@ -558,7 +558,7 @@ const layerScheduledTaskWebhookOrigin = Layer.effect(
558
559 const layerOrchestrationV2Runtime = RuntimeLayer.layerProduction.pipe(
560 Layer.provide(layerScheduledTaskWebhookOrigin),
561 - Layer.provide(ProviderEventIngestor.layerAnalytics),
561 + Layer.provide(ProviderUsageAnalytics.layer),
562 Layer.provide(layerCheckpointStore),
563 Layer.provide(layerGitWorkflow),
564 Layer.provide(
apps/server/src/telemetry/ProviderDimensions.ts
+40
@@ -4,6 +4,8 @@ import type {
4 ServerProvider,
5 } from "@t3tools/contracts";
6
7 +import { BUNDLED_MODEL_MANIFEST } from "../provider/ModelManifest.ts";
8 +
9 /**
10 * Drivers whose model catalogs are published by the vendor. Other drivers
11 * (OpenCode, Pi, ACP agents) list models from the user's own configuration,
@@ -39,6 +41,44 @@ export function providerDimensions(
41 : { provider: provider.driver };
42 }
43
44 +const DATED_SNAPSHOT = /-\d{8}$/u;
45 +
46 +// The bundled manifest is reviewed in this repo. Codex lists whatever its configured
47 +// model provider serves, so only the manifest vouches for Codex, Claude, and Antigravity.
48 +const MANIFEST_LABELS: ReadonlyMap<string, string> = new Map(
49 + Object.values(BUNDLED_MODEL_MANIFEST.providers ?? {}).flatMap((catalog) =>
50 + catalog.models.flatMap((model) =>
51 + [model.slug, ...(model.aliases ?? [])].map((name) => [name, model.slug] as const),
52 + ),
53 + ),
54 +);
55 +const REGISTRY_LABEL_DRIVERS: ReadonlySet<string> = new Set(["cursor", "grok"]);
56 +
57 +/**
58 + * The vendor catalog slug that a native or routed model id names, such as
59 + * `anthropic/claude-opus-5-5` or a dated snapshot. Models outside the bundled
60 + * manifest and the Cursor and Grok catalogs, including user-added ones, return undefined.
61 + */
62 +export function catalogModelLabel(
63 + providers: ReadonlyArray<ServerProvider>,
64 + model: string,
65 +): string | undefined {
66 + const leaf = model.slice(model.lastIndexOf("/") + 1);
67 + const candidates = [model, leaf, leaf.replace(DATED_SNAPSHOT, "")];
68 + for (const candidate of candidates) {
69 + const label = MANIFEST_LABELS.get(candidate);
70 + if (label !== undefined) return label;
71 + }
72 + for (const provider of providers) {
73 + if (!REGISTRY_LABEL_DRIVERS.has(provider.driver)) continue;
74 + const match = provider.models.find(
75 + (entry) => !entry.isCustom && candidates.includes(entry.slug),
76 + );
77 + if (match !== undefined) return match.slug;
78 + }
79 + return undefined;
80 +}
81 +
82 /** The thread fields agent analytics reads. Ids are used for lookups, never reported. */
83 export type AgentThread = Pick<
84 OrchestrationV2ThreadShell,
apps/server/src/telemetry/ProviderUsageAnalytics.test.ts new
+323
@@ -0,0 +1,323 @@
1 +import {
2 + NodeId,
3 + ProviderInstanceId,
4 + ProviderThreadId,
5 + ProviderTurnId,
6 + RunId,
7 + type OrchestrationV2ProviderTurn,
8 + type ServerProvider,
9 + type TurnTokenUsage,
10 +} from "@t3tools/contracts";
11 +import { describe, expect, it } from "@effect/vitest";
12 +import * as DateTime from "effect/DateTime";
13 +import * as Effect from "effect/Effect";
14 +import * as Layer from "effect/Layer";
15 +
16 +import * as ProviderRegistry from "../provider/ProviderRegistry.ts";
17 +import * as AnalyticsService from "./AnalyticsService.ts";
18 +import { catalogModelLabel } from "./ProviderDimensions.ts";
19 +import {
20 + ProviderTurnAnalytics,
21 + RequestUsageLedger,
22 + layer,
23 + providerTurnProperties,
24 + type ProviderTurnAnalyticsContext,
25 + type ProviderTurnSegment,
26 +} from "./ProviderUsageAnalytics.ts";
27 +
28 +const providers = [
29 + // Codex lists whatever its configured model provider serves, including private deployments.
30 + { driver: "codex", models: [{ slug: "gpt-6-sol" }, { slug: "acme-azure-deployment" }] },
31 + { driver: "claudeAgent", models: [{ slug: "claude-opus-5-5" }, { slug: "claude-haiku-4-5" }] },
32 + { driver: "opencode", models: [{ slug: "ollama/acme-internal" }] },
33 +].map(
34 + ({ driver, models }) =>
35 + ({
36 + instanceId: ProviderInstanceId.make(driver),
37 + driver,
38 + models: models.map((model) => ({ ...model, isCustom: false })),
39 + }) as unknown as ServerProvider,
40 +);
41 +const label = (model: string) => catalogModelLabel(providers, model);
42 +
43 +const at = (ms: number) => DateTime.makeUnsafe(ms);
44 +const complete = (input: number, output: number, extra: Partial<TurnTokenUsage> = {}) =>
45 + ({
46 + usageStatus: "complete",
47 + usageScope: "main_agent",
48 + hasSubagents: false,
49 + inputTokens: input,
50 + outputTokens: output,
51 + ...extra,
52 + }) as TurnTokenUsage;
53 +
54 +const turn = (
55 + id: string,
56 + usage: TurnTokenUsage | undefined,
57 + status: OrchestrationV2ProviderTurn["status"] = "completed",
58 +): OrchestrationV2ProviderTurn => ({
59 + id: ProviderTurnId.make(id),
60 + providerThreadId: ProviderThreadId.make("provider-thread"),
61 + nodeId: NodeId.make(`node:${id}`),
62 + runAttemptId: null,
63 + nativeTurnRef: null,
64 + ordinal: 1,
65 + status,
66 + startedAt: at(1_000),
67 + completedAt: at(4_000),
68 + ...(usage === undefined ? {} : { turnTokenUsage: usage }),
69 +});
70 +
71 +const context = (
72 + runId: string,
73 + segment: ProviderTurnSegment,
74 + model = "gpt-6-sol",
75 +): ProviderTurnAnalyticsContext => ({
76 + runId: RunId.make(runId),
77 + modelSelection: {
78 + instanceId: ProviderInstanceId.make("codex"),
79 + model,
80 + options: [{ id: "reasoningEffort", value: "xhigh" }],
81 + },
82 + runtimeMode: "full-access",
83 + interactionMode: "default",
84 + origin: { createdBy: "user", creationSource: "web", scheduled: false, subagent: false },
85 + segment,
86 +});
87 +
88 +const settled = (runId: string, childrenRunning = false) => ({
89 + runId: RunId.make(runId),
90 + status: "completed" as const,
91 + completedAt: at(9_000),
92 + childrenRunning,
93 +});
94 +
95 +describe("provider usage reporting", () => {
96 + it.effect(
97 + "records one request across a continuation, a child, a replay, and a late child",
98 + () => {
99 + const recorded: Array<readonly [string, unknown]> = [];
100 + const layerTest = layer.pipe(
101 + Layer.provide(
102 + Layer.mergeAll(
103 + Layer.mock(AnalyticsService.AnalyticsService)({
104 + record: (event, properties) =>
105 + Effect.sync(() => void recorded.push([event, properties])),
106 + }),
107 + Layer.mock(ProviderRegistry.ProviderRegistry)({
108 + getProviders: Effect.succeed(providers),
109 + }),
110 + ),
111 + ),
112 + );
113 + return Effect.gen(function* () {
114 + const analytics = yield* ProviderTurnAnalytics;
115 + const root = context("run-a", "initial");
116 + yield* analytics.runStarted({ runId: RunId.make("run-a"), driver: "codex", context: root });
117 + const first = { driver: "codex", context: root };
118 + const firstTurn = turn("root", complete(100, 10, { models: ["gpt-6-sol"] }));
119 + yield* analytics.turnCompleted({ ...first, providerTurn: firstTurn });
120 + yield* analytics.turnCompleted({ ...first, providerTurn: firstTurn });
121 + yield* analytics.turnCompleted({
122 + driver: "codex",
123 + context: context("run-a", "continuation"),
124 + providerTurn: turn("goal", complete(50, 5, { models: ["gpt-6-sol"] })),
125 + });
126 + yield* analytics.turnCompleted({
127 + driver: "codex",
128 + context: context("run-a", "native_child"),
129 + providerTurn: turn("child", complete(20, 2)),
130 + });
131 + yield* analytics.runSettled(settled("run-a"));
132 + yield* analytics.turnCompleted({
133 + driver: "codex",
134 + context: context("run-a", "native_child"),
135 + providerTurn: turn("late-child", complete(7, 1)),
136 + });
137 + yield* analytics.runSettled(settled("run-a"));
138 + yield* analytics.runStarted({
139 + runId: RunId.make("run-b"),
140 + driver: "codex",
141 + context: context("run-b", "initial"),
142 + });
143 + yield* analytics.runSettled(settled("run-b"));
144 +
145 + const requests = recorded.filter(([event]) => event === "provider.request.completed");
146 + expect(recorded.filter(([event]) => event === "provider.turn.completed")).toHaveLength(5);
147 + expect(requests).toHaveLength(2);
148 + // The whole payload: no run, turn, or thread ids, prompts, or raw model names.
149 + expect(requests[0]?.[1]).toEqual({
150 + reportVersion: 2,
151 + provider: "codex",
152 + requestKind: "user",
153 + terminalStatus: "completed",
154 + lifecycleObserved: true,
155 + model: "gpt-6-sol",
156 + effort: "xhigh",
157 + runtimeMode: "full-access",
158 + interactionMode: "default",
159 + createdBy: "user",
160 + creationSource: "web",
161 + scheduled: false,
162 + subagent: false,
163 + providerTurns: 2,
164 + continuations: 1,
165 + steeringRestarts: 0,
166 + usageStatus: "complete",
167 + inputTokens: 150,
168 + outputTokens: 15,
169 + nativeChildTurns: 1,
170 + childUsageStatus: "complete",
171 + childInputTokens: 20,
172 + childOutputTokens: 2,
173 + durationMs: 8_000,
174 + });
175 + expect(requests[1]?.[1]).toMatchObject({ providerTurns: 0, usageStatus: "unavailable" });
176 + }).pipe(Effect.provide(layerTest));
177 + },
178 + );
179 +
180 + it("keeps a request partial when its start, a steered segment, or a counter is missing", () => {
181 + const ledger = new RequestUsageLedger();
182 + ledger.start(RunId.make("restarted"), "codex", {
183 + ...context("restarted", "initial"),
184 + restartContinuation: true,
185 + });
186 + ledger.add({
187 + driver: "codex",
188 + context: context("restarted", "initial"),
189 + providerTurn: turn("after-restart", complete(10, 1)),
190 + });
191 + expect(ledger.settle(settled("restarted"), label)).toMatchObject({
192 + restartContinuation: true,
193 + lifecycleObserved: false,
194 + usageStatus: "partial",
195 + inputTokens: 10,
196 + });
197 +
198 + ledger.start(RunId.make("steered"), "claudeAgent", context("steered", "initial"));
199 + ledger.add({
200 + driver: "claudeAgent",
201 + context: context("steered", "initial"),
202 + providerTurn: turn(
203 + "steered-away",
204 + { usageStatus: "partial", usageScope: "main_agent", hasSubagents: false, outputTokens: 4 },
205 + "interrupted",
206 + ),
207 + });
208 + ledger.add({
209 + driver: "claudeAgent",
210 + context: context("steered", "steering_restart"),
211 + providerTurn: turn("steered-to", complete(30, 6)),
212 + });
213 + expect(ledger.settle(settled("steered"), label)).toMatchObject({
214 + usageStatus: "partial",
215 + steeringRestarts: 1,
216 + inputTokens: 30,
217 + outputTokens: 10,
218 + });
219 +
220 + ledger.start(RunId.make("silent"), "grok", context("silent", "initial"));
221 + ledger.add({
222 + driver: "grok",
223 + context: context("silent", "initial"),
224 + providerTurn: turn("silent", undefined),
225 + });
226 + const silent = ledger.settle(settled("silent", true), label);
227 + expect(silent).toMatchObject({
228 + usageStatus: "unavailable",
229 + childrenRunning: true,
230 + childUsageStatus: "unavailable",
231 + });
232 + expect(silent).not.toHaveProperty("outputTokens");
233 + });
234 +
235 + it("confirms only the models a provider reported", () => {
236 + const opus = context("models", "initial", "claude-opus-5-5");
237 + const properties = (usage: TurnTokenUsage | undefined) =>
238 + providerTurnProperties(
239 + { driver: "claudeAgent", context: opus, providerTurn: turn("t", usage) },
240 + label,
241 + );
242 + expect(properties(complete(1, 1, { models: ["claude-haiku-4-5-20251001"] }))).toMatchObject({
243 + model: "claude-opus-5-5",
244 + observedModel: "claude-haiku-4-5",
245 + mixedModels: false,
246 + });
247 + const unconfirmed = properties(complete(1, 1));
248 + expect(unconfirmed).toMatchObject({ model: "claude-opus-5-5" });
249 + expect(unconfirmed).not.toHaveProperty("mixedModels");
250 + expect(properties(complete(1, 1, { models: ["a", "b"] }))).toMatchObject({ mixedModels: true });
251 +
252 + const ledger = new RequestUsageLedger();
253 + ledger.start(RunId.make("mixed"), "codex", context("mixed", "initial"));
254 + ledger.add({
255 + driver: "codex",
256 + context: context("mixed", "initial"),
257 + providerTurn: turn("root", complete(1, 1, { models: ["gpt-6-sol"] })),
258 + });
259 + ledger.add({
260 + driver: "codex",
261 + context: context("mixed", "native_child"),
262 + providerTurn: turn("child", complete(1, 1, { models: ["gpt-6-luna"] })),
263 + });
264 + expect(ledger.settle(settled("mixed"), label)).toMatchObject({ mixedModels: true });
265 + });
266 +
267 + it("keeps zero, absent, local-command, and ledger usage distinct", () => {
268 + const properties = (usage: TurnTokenUsage | undefined) =>
269 + providerTurnProperties(
270 + { driver: "claudeAgent", context: context("z", "initial"), providerTurn: turn("t", usage) },
271 + label,
272 + );
273 + expect(properties(complete(0, 0))).toMatchObject({
274 + usageStatus: "complete",
275 + usageReported: true,
276 + inputTokens: 0,
277 + outputTokens: 0,
278 + });
279 + const absent = properties(undefined);
280 + expect(absent).toMatchObject({ usageStatus: "unavailable", usageReported: false });
281 + expect(absent).not.toHaveProperty("outputTokens");
282 + expect(
283 + properties(
284 + complete(0, 0, { localCommand: true, otherUsage: { inputTokens: 900, outputTokens: 80 } }),
285 + ),
286 + ).toMatchObject({ localCommand: true, otherInputTokens: 900, otherOutputTokens: 80 });
287 + expect(properties(complete(74_167, 236, { reasoningTokens: 30 }))).toMatchObject({
288 + outputTokens: 236,
289 + reasoningTokens: 30,
290 + });
291 + });
292 +
293 + it("reports routed catalog models and closed efforts only", () => {
294 + const routed = (model: string, variant: string) =>
295 + providerTurnProperties(
296 + {
297 + driver: "opencode",
298 + context: {
299 + modelSelection: {
300 + instanceId: ProviderInstanceId.make("opencode"),
301 + model,
302 + options: [{ id: "variant", value: variant }],
303 + },
304 + },
305 + providerTurn: turn("t", complete(1, 1, { models: [model] })),
306 + },
307 + label,
308 + );
309 + expect(routed("my-proxy/claude-opus-5-5", "high")).toMatchObject({
310 + model: "claude-opus-5-5",
311 + observedModel: "claude-opus-5-5",
312 + effort: "high",
313 + });
314 + for (const local of [
315 + routed("ollama/acme-internal", "secret-variant"),
316 + routed("acme-azure-deployment", "high"),
317 + ]) {
318 + expect(local).not.toHaveProperty("model");
319 + expect(local).not.toHaveProperty("observedModel");
320 + }
321 + expect(routed("ollama/acme-internal", "secret-variant")).not.toHaveProperty("effort");
322 + });
323 +});
apps/server/src/telemetry/ProviderUsageAnalytics.ts new
+407
@@ -0,0 +1,407 @@
1 +/**
2 + * Provider token reporting: one `provider.turn.completed` per provider turn and one
3 + * `provider.request.completed` per run, built from the same turns so each counts once.
4 + *
5 + * @module ProviderUsageAnalytics
6 + */
7 +import type {
8 + ModelSelection,
9 + OrchestrationV2Actor,
10 + OrchestrationV2CreationSource,
11 + OrchestrationV2ProviderTurn,
12 + OrchestrationV2RunAttempt,
13 + ProviderInteractionMode,
14 + RunId,
15 + RuntimeMode,
16 + TurnTokenUsage,
17 +} from "@t3tools/contracts";
18 +import * as Context from "effect/Context";
19 +import * as DateTime from "effect/DateTime";
20 +import * as Effect from "effect/Effect";
21 +import * as Layer from "effect/Layer";
22 +
23 +import { getModelSelectionStringOptionValue } from "@t3tools/shared/model";
24 +import * as ProviderRegistry from "../provider/ProviderRegistry.ts";
25 +import * as AnalyticsService from "./AnalyticsService.ts";
26 +import { catalogModelLabel } from "./ProviderDimensions.ts";
27 +
28 +// Bump when a field changes meaning, so reports can split old and new rows.
29 +const REPORT_VERSION = 2;
30 +
31 +/**
32 + * Where the run's starting message came from. Most turns start without a
33 + * person (provider continuations, agent sends, delegated subagents), so
34 + * reports need this to tell user turns apart.
35 + */
36 +export interface ProviderTurnOrigin {
37 + readonly createdBy: OrchestrationV2Actor;
38 + readonly creationSource: OrchestrationV2CreationSource;
39 + readonly scheduled: boolean;
40 + readonly subagent: boolean;
41 +}
42 +
43 +/** What started this provider turn inside its run. */
44 +export type ProviderTurnSegment =
45 + | OrchestrationV2RunAttempt["reason"]
46 + | "continuation"
47 + | "native_child";
48 +
49 +export interface ProviderTurnAnalyticsContext {
50 + readonly runId?: RunId;
51 + readonly modelSelection: ModelSelection;
52 + readonly runtimeMode?: RuntimeMode;
53 + readonly interactionMode?: ProviderInteractionMode;
54 + readonly origin?: ProviderTurnOrigin;
55 + readonly segment?: ProviderTurnSegment;
56 + readonly restartContinuation?: boolean;
57 +}
58 +
59 +export interface ProviderTurnReport {
60 + readonly driver: string;
61 + readonly providerTurn: OrchestrationV2ProviderTurn;
62 + readonly context?: ProviderTurnAnalyticsContext;
63 +}
64 +
65 +export interface RunSettledReport {
66 + readonly runId: RunId;
67 + readonly status: OrchestrationV2ProviderTurn["status"];
68 + readonly completedAt: DateTime.Utc;
69 + // Run-owned children were still open when the run settled; their later turns are not added.
70 + readonly childrenRunning: boolean;
71 +}
72 +
73 +type Properties = Readonly<Record<string, unknown>>;
74 +/** Maps a native model id to a vendor catalog label, or undefined to keep it out. */
75 +export type ModelLabeler = (model: string) => string | undefined;
76 +
77 +const TERMINAL_STATUSES: ReadonlySet<OrchestrationV2ProviderTurn["status"]> = new Set([
78 + "completed",
79 + "failed",
80 + "interrupted",
81 + "cancelled",
82 +]);
83 +
84 +const REPORTED_EFFORTS: ReadonlySet<string> = new Set([
85 + "none",
86 + "minimal",
87 + "low",
88 + "medium",
89 + "high",
90 + "xhigh",
91 + "max",
92 + "ultrathink",
93 + "ultracode",
94 + "brutal",
95 +]);
96 +
97 +const defined = (entries: Properties): Properties =>
98 + Object.fromEntries(Object.entries(entries).filter(([, value]) => value !== undefined));
99 +
100 +// OpenCode names its effort option `variant`, and variants can be user-defined.
101 +function selectedEffort(selection: ModelSelection): string | undefined {
102 + const effort =
103 + getModelSelectionStringOptionValue(selection, "reasoningEffort") ??
104 + getModelSelectionStringOptionValue(selection, "effort") ??
105 + getModelSelectionStringOptionValue(selection, "variant");
106 + return effort !== undefined && REPORTED_EFFORTS.has(effort) ? effort : undefined;
107 +}
108 +
109 +/**
110 + * Confirmed models across counted work. Unknown when any piece of work has no
111 + * reported model, unless two distinct models were already seen.
112 + */
113 +function modelObservation(
114 + models: ReadonlyArray<ReadonlyArray<string> | undefined>,
115 + label: ModelLabeler,
116 +): Properties {
117 + const distinct = new Set(models.flatMap((entry) => entry ?? []));
118 + if (distinct.size > 1) return { mixedModels: true };
119 + if (distinct.size === 0 || models.some((entry) => entry === undefined || entry.length === 0)) {
120 + return {};
121 + }
122 + return defined({ mixedModels: false, observedModel: label([...distinct][0]!) });
123 +}
124 +
125 +function selectionProperties(
126 + context: ProviderTurnAnalyticsContext | undefined,
127 + label: ModelLabeler,
128 +) {
129 + const selection = context?.modelSelection;
130 + return defined({
131 + model: selection === undefined ? undefined : label(selection.model),
132 + effort: selection === undefined ? undefined : selectedEffort(selection),
133 + runtimeMode: context?.runtimeMode,
134 + interactionMode: context?.interactionMode,
135 + ...context?.origin,
136 + });
137 +}
138 +
139 +function childKind(context: ProviderTurnAnalyticsContext | undefined) {
140 + if (context?.segment === "native_child") return "native";
141 + return context?.origin?.subagent === true ? "t3" : undefined;
142 +}
143 +
144 +function counters(usage: TurnTokenUsage | undefined): Properties {
145 + return defined({
146 + inputTokens: usage?.inputTokens,
147 + cachedInputTokens: usage?.cachedInputTokens,
148 + cacheCreationTokens: usage?.cacheCreationTokens,
149 + outputTokens: usage?.outputTokens,
150 + reasoningTokens: usage?.reasoningTokens,
151 + });
152 +}
153 +
154 +export function providerTurnProperties(
155 + report: ProviderTurnReport,
156 + label: ModelLabeler,
157 +): Properties {
158 + const { providerTurn, context } = report;
159 + const usage = providerTurn.turnTokenUsage;
160 + return defined({
161 + reportVersion: REPORT_VERSION,
162 + provider: report.driver,
163 + terminalStatus: providerTurn.status,
164 + usageStatus: usage?.usageStatus ?? "unavailable",
165 + usageScope: usage?.usageScope ?? "main_agent",
166 + usageReported: usage !== undefined,
167 + hasSubagents: usage?.hasSubagents,
168 + ...counters(usage),
169 + localCommand: usage?.localCommand === true ? true : undefined,
170 + otherInputTokens: usage?.otherUsage?.inputTokens,
171 + otherOutputTokens: usage?.otherUsage?.outputTokens,
172 + ...selectionProperties(context, label),
173 + ...modelObservation([usage?.models], label),
174 + segment: context?.segment,
175 + childKind: childKind(context),
176 + durationMs:
177 + providerTurn.startedAt && providerTurn.completedAt
178 + ? Math.max(
179 + 0,
180 + DateTime.toEpochMillis(providerTurn.completedAt) -
181 + DateTime.toEpochMillis(providerTurn.startedAt),
182 + )
183 + : undefined,
184 + });
185 +}
186 +
187 +/** How the request reached the provider; only `user` counts as a person asking. */
188 +function requestKind(origin: ProviderTurnOrigin | undefined) {
189 + if (origin === undefined) return "unknown";
190 + if (origin.scheduled) return "scheduled";
191 + if (origin.subagent) return "t3_child";
192 + if (origin.createdBy === "user") return "user";
193 + if (origin.creationSource === "provider") return "provider_continuation";
194 + return origin.createdBy;
195 +}
196 +
197 +function sumOf(
198 + usages: ReadonlyArray<TurnTokenUsage>,
199 + field:
200 + | "inputTokens"
201 + | "cachedInputTokens"
202 + | "cacheCreationTokens"
203 + | "outputTokens"
204 + | "reasoningTokens",
205 + requireAll: boolean,
206 +): number | undefined {
207 + const values = usages.map((usage) => usage[field]);
208 + const known = values.filter((value) => value !== undefined);
209 + if (known.length === 0 || (requireAll && known.length < values.length)) return undefined;
210 + return known.reduce((sum, value) => sum + value, 0);
211 +}
212 +
213 +/** complete only when every turn is complete; partial when any turn has counts. */
214 +function combinedStatus(usages: ReadonlyArray<TurnTokenUsage | undefined>) {
215 + if (usages.length > 0 && usages.every((usage) => usage?.usageStatus === "complete")) {
216 + return "complete";
217 + }
218 + return usages.some(
219 + (usage) => usage?.inputTokens !== undefined || usage?.outputTokens !== undefined,
220 + )
221 + ? "partial"
222 + : "unavailable";
223 +}
224 +
225 +interface RequestTurn {
226 + readonly child: boolean;
227 + readonly segment: ProviderTurnSegment | undefined;
228 + readonly providerTurn: OrchestrationV2ProviderTurn;
229 +}
230 +
231 +interface RequestEntry {
232 + driver: string | undefined;
233 + context: ProviderTurnAnalyticsContext | undefined;
234 + readonly lifecycleObserved: boolean;
235 + readonly turns: Map<string, RequestTurn>;
236 +}
237 +
238 +const MAX_TRACKED_RUNS = 2048;
239 +
240 +/**
241 + * Collects terminal provider turns per run in this process. A replayed turn
242 + * replaces its earlier copy, a settled run ignores late turns, and a run whose
243 + * initial attempt started before this process reports `lifecycleObserved: false`.
244 + */
245 +export class RequestUsageLedger {
246 + private readonly open = new Map<RunId, RequestEntry>();
247 + private readonly settled = new Set<RunId>();
248 +
249 + start(runId: RunId, driver: string, context: ProviderTurnAnalyticsContext): void {
250 + if (this.settled.has(runId) || this.open.has(runId)) return;
251 + // A restart continuation carries on a run whose earlier turns this process never saw.
252 + const lifecycleObserved = context.restartContinuation !== true;
253 + this.open.set(runId, { driver, context, lifecycleObserved, turns: new Map() });
254 + evictOldest(this.open);
255 + }
256 +
257 + add(report: ProviderTurnReport): void {
258 + const runId = report.context?.runId;
259 + if (runId === undefined || this.settled.has(runId)) return;
260 + if (!TERMINAL_STATUSES.has(report.providerTurn.status)) return;
261 + let entry = this.open.get(runId);
262 + if (entry === undefined) {
263 + entry = { driver: undefined, context: undefined, lifecycleObserved: false, turns: new Map() };
264 + this.open.set(runId, entry);
265 + evictOldest(this.open);
266 + }
267 + const child = report.context?.segment === "native_child";
268 + if (!child) {
269 + entry.driver ??= report.driver;
270 + entry.context ??= report.context;
271 + }
272 + entry.turns.set(String(report.providerTurn.id), {
273 + child,
274 + segment: report.context?.segment,
275 + providerTurn: report.providerTurn,
276 + });
277 + }
278 +
279 + settle(report: RunSettledReport, label: ModelLabeler): Properties | undefined {
280 + if (this.settled.has(report.runId)) return undefined;
281 + this.settled.add(report.runId);
282 + evictOldest(this.settled);
283 + const entry = this.open.get(report.runId);
284 + this.open.delete(report.runId);
285 + if (entry === undefined) return undefined;
286 +
287 + const turns = [...entry.turns.values()];
288 + const root = turns.filter((turn) => !turn.child);
289 + const children = turns.filter((turn) => turn.child);
290 + const rootUsage = root.map((turn) => turn.providerTurn.turnTokenUsage);
291 + const childUsage = children.map((turn) => turn.providerTurn.turnTokenUsage);
292 + const { lifecycleObserved } = entry;
293 + const measuredRoot = rootUsage.filter((usage) => usage !== undefined);
294 + const status = combinedStatus(rootUsage);
295 + const rootComplete = status === "complete";
296 + const otherUsage = measuredRoot.map((usage) => usage.otherUsage);
297 + const allOtherKnown = otherUsage.length > 0 && otherUsage.every((usage) => usage !== undefined);
298 + const anyRootSubagents = measuredRoot.some((usage) => usage.hasSubagents);
299 + const measuredChildren = childUsage.filter((usage) => usage !== undefined);
300 + const childStatus =
301 + children.length === 0 && !anyRootSubagents && !report.childrenRunning
302 + ? "none"
303 + : children.length === 0
304 + ? "unavailable"
305 + : report.childrenRunning && combinedStatus(childUsage) === "complete"
306 + ? "partial"
307 + : combinedStatus(childUsage);
308 + const startedAt = turns
309 + .map((turn) => turn.providerTurn.startedAt)
310 + .filter((value) => value !== null)
311 + .map(DateTime.toEpochMillis);
312 +
313 + return defined({
314 + reportVersion: REPORT_VERSION,
315 + provider: entry.driver ?? "unknown",
316 + requestKind: requestKind(entry.context?.origin),
317 + restartContinuation: entry.context?.restartContinuation === true ? true : undefined,
318 + terminalStatus: report.status,
319 + lifecycleObserved,
320 + ...selectionProperties(entry.context, label),
321 + providerTurns: root.length,
322 + continuations: root.filter((turn) => turn.segment === "continuation").length,
323 + steeringRestarts: root.filter((turn) => turn.segment === "steering_restart").length,
324 + usageStatus: lifecycleObserved || status === "unavailable" ? status : "partial",
325 + inputTokens: sumOf(measuredRoot, "inputTokens", rootComplete),
326 + cachedInputTokens: sumOf(measuredRoot, "cachedInputTokens", true),
327 + cacheCreationTokens: sumOf(measuredRoot, "cacheCreationTokens", true),
328 + outputTokens: sumOf(measuredRoot, "outputTokens", rootComplete),
329 + reasoningTokens: sumOf(measuredRoot, "reasoningTokens", true),
330 + localCommand: measuredRoot.some((usage) => usage.localCommand === true) ? true : undefined,
331 + otherInputTokens: allOtherKnown
332 + ? otherUsage.reduce((sum, usage) => sum + (usage?.inputTokens ?? 0), 0)
333 + : undefined,
334 + otherOutputTokens: allOtherKnown
335 + ? otherUsage.reduce((sum, usage) => sum + (usage?.outputTokens ?? 0), 0)
336 + : undefined,
337 + nativeChildTurns: children.length,
338 + childUsageStatus: childStatus,
339 + childInputTokens: sumOf(measuredChildren, "inputTokens", false),
340 + childOutputTokens: sumOf(measuredChildren, "outputTokens", false),
341 + childrenRunning: report.childrenRunning ? true : undefined,
342 + ...modelObservation(
343 + [...rootUsage, ...childUsage].map((usage) => usage?.models),
344 + label,
345 + ),
346 + durationMs:
347 + startedAt.length === 0
348 + ? undefined
349 + : Math.max(0, DateTime.toEpochMillis(report.completedAt) - Math.min(...startedAt)),
350 + });
351 + }
352 +}
353 +
354 +function evictOldest(collection: Map<RunId, unknown> | Set<RunId>) {
355 + if (collection.size <= MAX_TRACKED_RUNS) return;
356 + const oldest = collection.keys().next().value;
357 + if (oldest !== undefined) collection.delete(oldest);
358 +}
359 +
360 +export class ProviderTurnAnalytics extends Context.Reference<{
361 + readonly runStarted: (input: {
362 + readonly runId: RunId;
363 + readonly driver: string;
364 + readonly context: ProviderTurnAnalyticsContext;
365 + }) => Effect.Effect<void>;
366 + readonly turnCompleted: (report: ProviderTurnReport) => Effect.Effect<void>;
367 + readonly runSettled: (report: RunSettledReport) => Effect.Effect<void>;
368 +}>("t3/telemetry/ProviderTurnAnalytics", {
369 + defaultValue: () => ({
370 + runStarted: () => Effect.void,
371 + turnCompleted: () => Effect.void,
372 + runSettled: () => Effect.void,
373 + }),
374 +}) {}
375 +
376 +export const layer = Layer.effect(
377 + ProviderTurnAnalytics,
378 + Effect.gen(function* () {
379 + const analytics = yield* AnalyticsService.AnalyticsService;
380 + const registry = yield* ProviderRegistry.ProviderRegistry;
381 + const ledger = new RequestUsageLedger();
382 + const labeler = Effect.map(
383 + registry.getProviders,
384 + (providers): ModelLabeler =>
385 + (model) =>
386 + catalogModelLabel(providers, model),
387 + );
388 + return {
389 + runStarted: Effect.fn("ProviderTurnAnalytics.runStarted")((input) =>
390 + Effect.sync(() => ledger.start(input.runId, input.driver, input.context)),
391 + ),
392 + turnCompleted: Effect.fn("ProviderTurnAnalytics.turnCompleted")(function* (report) {
393 + ledger.add(report);
394 + yield* analytics.record(
395 + "provider.turn.completed",
396 + providerTurnProperties(report, yield* labeler),
397 + );
398 + }),
399 + runSettled: Effect.fn("ProviderTurnAnalytics.runSettled")(function* (report) {
400 + const properties = ledger.settle(report, yield* labeler);
401 + if (properties !== undefined) {
402 + yield* analytics.record("provider.request.completed", properties);
403 + }
404 + }),
405 + };
406 + }),
407 +);
docs/internals/product-analytics.md
+55 -16
@@ -53,22 +53,61 @@ A command whose id was already accepted is a retry, so it records nothing again.
53 One identity can appear in several client groups during a period; adding those
54 groups double-counts users.
55
56 -Provider send and completion counts need not match. Most completions start
57 -without a person: provider continuations, agent sends, and delegated subagents.
58 -`provider.turn.completed` carries the starting message's `createdBy` and
59 -`creationSource`, plus `scheduled` and `subagent`; filter on them to count user
60 -turns. Collection is best effort, with no scan or backfill of provider history.
61 -
62 -Token totals cover the main agent. Child agents and model rerouting prevent a turn
63 -from representing one provider/model combination's full cost. For provider
64 -comparisons, require complete usage, no observed subagents, and no mixed models;
65 -compare matching model, effort, interaction mode, and terminal status. Aggregate
66 -output/input ratios should divide the summed totals. Averaging per-turn ratios
67 -lets small-input turns dominate.
68 -
69 -Unknown counts stay absent. Partial usage contains valid observed counts but
70 -cannot establish a whole-turn total. Keep these distinctions when changing token
71 -normalization or building reports.
56 +Provider send and completion counts need not match. `provider.turn.completed`
57 +is one provider turn, and `segment` says what started it: the run's first
58 +attempt (`initial`), a `steering_restart`, a provider `continuation` such as a
59 +Codex goal turn, or a `native_child` subagent. `createdBy`, `creationSource`,
60 +`scheduled`, and `subagent` describe the run's starting message, and `childKind`
61 +separates native subagents from T3-managed child threads. Use this event for
62 +per-turn measurements, not for counting requests.
63 +
64 +`provider.request.completed` is one run, recorded when the run settles in the
65 +server process that ran it. It adds each root turn once, so continuations,
66 +steering, and replayed provider events do not add rows. `requestKind: "user"`
67 +counts what people asked for. Native subagent turns are summed apart from the
68 +root (`childUsageStatus`, `childInputTokens`); T3-managed child threads report
69 +their own requests as `t3_child`. A child still running at settlement is never
70 +added later, and the summary sets `childrenRunning`. A server restart loses the
71 +summary of a run in progress. A run that continues after a restart reports its
72 +own usage with `restartContinuation` and `lifecycleObserved: false`; it cannot
73 +recover the original request's count or kind. A run whose provider session never
74 +opens has no summary, and a start the provider rejects reports
75 +`providerTurns: 0`. A Stop that lands while a steering restart is starting ends
76 +the run without a summary. Collection is best effort, with no scan or backfill of
77 +provider history.
78 +
79 +Counts cover the main agent. `otherInputTokens` and `otherOutputTokens` are what
80 +a provider's own ledger recorded beyond it during the turn: Claude's per-process
81 +`modelUsage` growth, which covers native subagents, compaction, and helper
82 +calls. They need a baseline from the same process, so they stay absent when it
83 +is unknown or the ledger goes backwards, and background work that ends between
84 +turns lands in the next one. Claude subagents have no turns of their own, so
85 +this never overlaps child turns.
86 +
87 +`model` and `effort` are what the run selected. `observedModel` and
88 +`mixedModels` appear only when the provider reported the models for the counted
89 +work (Codex thread settings and reroutes, Claude root messages, OpenCode steps).
90 +`mixedModels: false` without `reportVersion` was never checked. Model labels
91 +come from the bundled model manifest and the Cursor and Grok catalogs, also for
92 +routed ids such as `anthropic/claude-opus-5-5`, and efforts come from a closed
93 +list, so user-configured names stay out.
94 +
95 +Status and counters answer different questions. `usageReported: false` means
96 +the adapter attached no usage; a harness where every row says so is unsupported,
97 +not idle. `partial` keeps valid counts that cannot establish a whole-turn total,
98 +such as an OpenCode step that ended without a report, an OpenCode stream
99 +reconnect, or an interrupted segment.
100 +Explicit zeros are sent and unknown counts stay absent. `localCommand` marks a
101 +provider-local command such as Claude's `/compact`, whose result counters are not
102 +model work. Reasoning tokens are a subset of output.
103 +
104 +Charts built on these events should say "output tokens per measured provider
105 +completion", compare matching provider, observed model, effort, interaction mode,
106 +and terminal status, and show eligible and measured counts by harness, origin,
107 +and status so unknown models and unsupported adapters stay in the denominator.
108 +Divide summed totals for output/input ratios; averaging per-turn ratios lets
109 +small-input turns dominate. Rows without `reportVersion` keep their missing
110 +fields unknown; a missing origin is never a person.
111
112 `feature.used` counts WebSocket requests that start a feature. RPCs are observed in
113 the RPC authorization middleware after the request succeeds; dispatched commands
docs/user/telemetry.md
+4 -3
@@ -5,11 +5,12 @@ associated with a hashed account or installation identifier. To count how many i
5 uses, the server also sends hashed account identifiers: your T3 Connect account, your OpenAI and
6 Claude accounts when you are signed in, and a random identifier for each device that connects. The endpoint keeps a copy of each
7 event and forwards events to PostHog. Events include the provider, model, reasoning effort,
8 -permission mode, turn result, duration, release channel, and main-agent token totals when
9 -available.
8 +permission mode, turn and request results, duration, release channel, and token counts when
9 +available. Child-agent token counts are reported separately from the main agent's. Model names
10 +are sent only when they appear in a provider's public model list.
11
12 Events do not include prompts, responses, file contents, authentication tokens, conversation IDs,
12 -raw provider events, or child-agent output. Child-agent token use is excluded from the totals.
13 +raw provider events, or child-agent output.
14
15 T3 Code also sends relay connection traces to Axiom to diagnose T3 Connect connectivity.
16
packages/contracts/src/providerRuntime.ts
+8
@@ -60,6 +60,14 @@ const TurnTokenUsageCommonFields = {
60 cacheCreationTokens: Schema.optional(NonNegativeInt),
61 reasoningTokens: Schema.optional(NonNegativeInt),
62 hasSubagents: Schema.Boolean,
63 + // Native model ids the provider reported for the main agent's work. Absent means unknown.
64 + models: Schema.optional(Schema.Array(TrimmedNonEmptyStringSchema)),
65 + // The provider ran a local command; its counters describe that result, not model work.
66 + localCommand: Schema.optional(Schema.Boolean),
67 + // Provider-ledger counts beyond the main agent, such as native subagents and compaction.
68 + otherUsage: Schema.optional(
69 + Schema.Struct({ inputTokens: NonNegativeInt, outputTokens: NonNegativeInt }),
70 + ),
71 };
72 export const TurnTokenUsage = Schema.Union([
73 Schema.Struct({
packages/provider-opencode/src/server/v2/adapter.ts
+27 -3
@@ -209,7 +209,12 @@ interface ActiveTurn {
209 output: number;
210 reasoning: number;
211 };
212 + // Steps OpenCode started; a step that ends without tokens leaves the total partial.
213 + startedSteps: number;
214 + // A reconnect dropped events, so steps may have ended unseen.
215 + missedEvents: boolean;
216 steps: number;
217 + readonly models: Set<string>;
218 lastStep: Tokens | undefined;
219 before: string | null | undefined;
220 compaction: { readonly nativeId: string; readonly startedAt: DateTime.Utc } | undefined;
@@ -629,16 +634,26 @@ const skillsNamed = (text: string, known: ReadonlySet<string>) => [
634 ),
635 ];
636
632 -const turnTokenUsage = (turn: ActiveTurn, status: OrchestrationV2ProviderTurn["status"]) =>
633 - turn.steps === 0
637 +const turnTokenUsage = (turn: ActiveTurn, status: OrchestrationV2ProviderTurn["status"]) => {
638 + // A step whose start was missed, in a reconnect gap or as an unreadable event, ran on an unreported model.
639 + const models =
640 + turn.models.size === 0 || turn.missedEvents || turn.steps > turn.startedSteps
641 + ? {}
642 + : { models: [...turn.models] };
643 + return turn.steps === 0
644 ? {
645 usageScope: "main_agent" as const,
646 usageStatus: "unavailable" as const,
647 hasSubagents: turn.usedSubagents,
648 + ...models,
649 }
650 : {
651 + ...models,
652 usageScope: "main_agent" as const,
641 - usageStatus: status === "completed" ? ("complete" as const) : ("partial" as const),
653 + usageStatus:
654 + status === "completed" && !turn.missedEvents && turn.steps >= turn.startedSteps
655 + ? ("complete" as const)
656 + : ("partial" as const),
657 inputTokens: turn.usage.input,
658 cachedInputTokens: turn.usage.cached,
659 cacheCreationTokens: turn.usage.cacheWrite,
@@ -646,6 +661,7 @@ const turnTokenUsage = (turn: ActiveTurn, status: OrchestrationV2ProviderTurn["s
661 reasoningTokens: turn.usage.reasoning,
662 hasSubagents: turn.usedSubagents,
663 };
664 +};
665
666 export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: ProviderInstanceId) {
667 const server = yield* OpenCode2Server.OpenCode2Server;
@@ -1111,7 +1127,10 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr
1127 ordinals: new Map(),
1128 nextOrdinal: providerTurn.ordinal * 100 + 1,
1129 usage: { input: 0, cached: 0, cacheWrite: 0, output: 0, reasoning: 0 },
1130 + startedSteps: 0,
1131 + missedEvents: false,
1132 steps: 0,
1133 + models: new Set(),
1134 lastStep: undefined,
1135 before: undefined,
1136 compaction: undefined,
@@ -2101,6 +2120,10 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr
2120 return yield* emitCompaction(state, turn, "completed", event.data.text);
2121 case "session.compaction.failed":
2122 return yield* emitCompaction(state, turn, "failed");
2123 + case "session.step.started":
2124 + turn.startedSteps += 1;
2125 + turn.models.add(`${event.data.model.providerID}/${event.data.model.id}`);
2126 + return;
2127 case "session.step.ended":
2128 case "session.step.failed": {
2129 const tokens = event.data.tokens;
@@ -2568,6 +2591,7 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr
2591 for (const [sessionId, state] of running) {
2592 const turn = state.active;
2593 if (turn === undefined) continue;
2594 + turn.missedEvents = true;
2595 const ended = yield* backfill(sessionId, state);
2596 if (sessionId in active || state.active !== turn) continue;
2597 yield* finishTurn(