@devries-cameron20 / t3code-1 / commits / 8bcef34f16
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): analytics stop counting reconnects, retries, and dev servers (#24601)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Theo Browne committed Oct 24, 2026 at 16:57 UTC 8bcef34f16ca2a08aa19c35dd64b71290bd4412f
15 files changed +330 -87
apps/server/src/orchestration-v2/Orchestrator.control-reads.test.ts
+2 -1
@@ -1285,7 +1285,8 @@ it.effect("replays a thread.create receipt without changing the thread", () =>
1285 const before = yield* projections.getThread(threadId);
1286 const replayed = yield* orchestrator.dispatch(create);
1287 assert.equal(replayed.sequence, original.sequence);
1288 - assert.deepEqual(replayed, original);
1288 + // Marked so client analytics do not count the retry again.
1289 + assert.deepEqual(replayed, { ...original, replayed: true });
1290 assert.deepEqual(yield* projections.getThread(threadId), before);
1291 assert.equal(yield* orchestrator.getThreadEventSequence(threadId), original.sequence);
1292 }).pipe(Effect.provide(layerTest)),
apps/server/src/orchestration-v2/Orchestrator.ts
+12 -4
@@ -295,6 +295,8 @@ export type OrchestratorV2Error = typeof OrchestratorV2Error.Type;
295 export interface OrchestratorV2DispatchResult {
296 readonly sequence: number;
297 readonly storedEvents: ReadonlyArray<OrchestrationV2StoredEvent>;
298 + /** The command id was already accepted; analytics skip it. */
299 + readonly replayed?: true;
300 }
301
302 export interface OrchestratorV2Shape {
@@ -11791,6 +11793,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
11793 return {
11794 sequence: receipt.resultSequence,
11795 storedEvents,
11796 + replayed: true,
11797 } satisfies OrchestratorV2DispatchResult;
11798 }
11799
@@ -11863,9 +11866,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
11866 if (plan.events.length === 0 && plan.effects.length === 0) {
11867 // A settle that ended nothing still records its receipt: a replayed Stop
11868 // effect then finds it instead of settling work that appeared since.
11866 - const resultSequence = yield* Effect.gen(function* () {
11869 + const { resultSequence, inserted } = yield* Effect.gen(function* () {
11870 const sequence = yield* eventSink.latestSequence({ threadId: commandThreadId(command) });
11868 - yield* commandReceipts.insertIfAbsent({
11871 + const inserted = yield* commandReceipts.insertIfAbsent({
11872 commandId: command.commandId,
11873 threadId: commandThreadId(command),
11874 commandType: command.type,
@@ -11874,7 +11877,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
11877 status: "accepted",
11878 error: null,
11879 });
11877 - return sequence;
11880 + return { resultSequence: sequence, inserted };
11881 }).pipe(
11882 Effect.mapError(
11883 (cause) =>
@@ -11885,7 +11888,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
11888 }),
11889 ),
11890 );
11888 - return { sequence: resultSequence, storedEvents: [] } satisfies OrchestratorV2DispatchResult;
11891 + return {
11892 + sequence: resultSequence,
11893 + storedEvents: [],
11894 + ...(inserted ? {} : { replayed: true as const }),
11895 + } satisfies OrchestratorV2DispatchResult;
11896 }
11897 const acceptedAt = plan.events.at(-1)?.occurredAt ?? (yield* DateTime.now);
11898 const committed = yield* eventSink
@@ -11936,6 +11943,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
11943 return {
11944 sequence: committed.receipt.resultSequence,
11945 storedEvents: committed.storedEvents,
11946 + ...(committed.committed ? {} : { replayed: true as const }),
11947 } satisfies OrchestratorV2DispatchResult;
11948 });
11949
apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts
+10
@@ -233,6 +233,12 @@ it.effect("records accepted billed turn usage once without billing the context w
233 modelSelection,
234 runtimeMode: "full-access" as const,
235 interactionMode: "default" as const,
236 + origin: {
237 + createdBy: "agent" as const,
238 + creationSource: "provider" as const,
239 + scheduled: false,
240 + subagent: true,
241 + },
242 },
243 event: { type: "provider_turn.updated" as const, driver: CODEX_DRIVER, providerTurn },
244 };
@@ -271,6 +277,10 @@ it.effect("records accepted billed turn usage once without billing the context w
277 mixedModels: false,
278 runtimeMode: "full-access",
279 interactionMode: "default",
280 + createdBy: "agent",
281 + creationSource: "provider",
282 + scheduled: false,
283 + subagent: true,
284 durationMs: 120,
285 });
286 }).pipe(Effect.provide(layerTest.pipe(Layer.provide(layerAnalytics))));
apps/server/src/orchestration-v2/ProviderEventIngestor.ts
+16
@@ -9,6 +9,8 @@ import {
9 type OrchestrationV2Subagent,
10 type OrchestrationV2ThreadProjection,
11 type ModelSelection,
12 + type OrchestrationV2Actor,
13 + type OrchestrationV2CreationSource,
14 type RuntimeMode,
15 type ProviderInteractionMode,
16 ProviderInstanceId,
@@ -77,6 +79,19 @@ export interface ProviderTurnAnalyticsContext {
79 readonly modelSelection: ModelSelection;
80 readonly runtimeMode?: RuntimeMode;
81 readonly interactionMode?: ProviderInteractionMode;
82 + readonly origin?: ProviderTurnOrigin;
83 +}
84 +
85 +/**
86 + * Where the run's starting message came from. Most turns start without a
87 + * person (provider continuations, agent sends, delegated subagents), so
88 + * reports need this to tell user turns apart.
89 + */
90 +export interface ProviderTurnOrigin {
91 + readonly createdBy: OrchestrationV2Actor;
92 + readonly creationSource: OrchestrationV2CreationSource;
93 + readonly scheduled: boolean;
94 + readonly subagent: boolean;
95 }
96
97 export class ProviderTurnAnalytics extends Context.Reference<{
@@ -126,6 +141,7 @@ function providerTurnAnalyticsProperties(input: {
141 ...(effort ? { effort } : {}),
142 ...(input.context?.runtimeMode ? { runtimeMode: input.context.runtimeMode } : {}),
143 ...(input.context?.interactionMode ? { interactionMode: input.context.interactionMode } : {}),
144 + ...input.context?.origin,
145 ...(input.providerTurn.startedAt && input.providerTurn.completedAt
146 ? {
147 durationMs: Math.max(
apps/server/src/orchestration-v2/RunExecutionService.test.ts
+35 -11
@@ -50,6 +50,11 @@ import * as RunFinalizationService from "./RunFinalizationService.ts";
50 import * as TurnItemPositionStore from "./TurnItemPositionStore.ts";
51
52 const driver = ProviderDriverKind.make("codex");
53 +const rootAppThread = (id: ThreadId) =>
54 + ({
55 + id,
56 + lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: id },
57 + }) as OrchestrationV2AppThread;
58
59 const layerTurnItemPositionsTest = Layer.mock(TurnItemPositionStore.TurnItemPositionStoreV2)({
60 allocate: () => Effect.succeed(1),
@@ -775,7 +780,7 @@ it.effect("rechecks run ownership immediately before calling the provider", () =
780
781 yield* runExecution.startRootRun({
782 commandId: CommandId.make("command:run-execution-start-guard"),
778 - appThread: { id: threadId } as OrchestrationV2AppThread,
783 + appThread: rootAppThread(threadId),
784 providerSessionId,
785 session,
786 run,
@@ -830,7 +835,7 @@ it.effect("passes the thread's MCP app context to the provider under a safe key"
835
836 yield* runExecution.startRootRun({
837 commandId: CommandId.make("command:run-execution-app-context"),
833 - appThread: { id: threadId } as OrchestrationV2AppThread,
838 + appThread: rootAppThread(threadId),
839 providerSessionId: ProviderSessionId.make("session:run-execution-app-context"),
840 session,
841 run: {
@@ -934,7 +939,7 @@ it.effect("fails the run when its ownership check cannot be read before calling
939 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
940 yield* runExecution.startRootRun({
941 commandId: CommandId.make("command:run-execution-start-guard-read"),
937 - appThread: { id: threadId } as OrchestrationV2AppThread,
942 + appThread: rootAppThread(threadId),
943 providerSessionId: ProviderSessionId.make("session:run-execution-start-guard-read"),
944 session: {
945 events: Stream.never,
@@ -1029,7 +1034,7 @@ it.effect(
1034 const attemptId = RunAttemptId.make(`attempt:compact-routing:${index}`);
1035 yield* runExecution.startRootRun({
1036 commandId: CommandId.make(`command:compact-routing:${index}`),
1032 - appThread: { id: threadId } as OrchestrationV2AppThread,
1037 + appThread: rootAppThread(threadId),
1038 providerSessionId: ProviderSessionId.make(`session:compact-routing:${index}`),
1039 session: {
1040 events: Stream.never,
@@ -1145,7 +1150,7 @@ it.effect("starts the provider when checkpoint baseline capture fails", () =>
1150 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
1151 yield* runExecution.startRootRun({
1152 commandId: CommandId.make("command:run-execution-baseline-failure"),
1148 - appThread: { id: threadId } as OrchestrationV2AppThread,
1153 + appThread: rootAppThread(threadId),
1154 providerSessionId,
1155 session: {
1156 events: Stream.never,
@@ -1283,7 +1288,7 @@ it.effect.each(["failure", "interruption", "stale-attempt", "start-guard"] as co
1288 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
1289 yield* runExecution.startRootRun({
1290 commandId: CommandId.make("command:run-execution-settings-failure"),
1286 - appThread: { id: threadId } as OrchestrationV2AppThread,
1291 + appThread: rootAppThread(threadId),
1292 providerSessionId,
1293 session: {
1294 events: Stream.never,
@@ -1399,6 +1404,7 @@ it.effect("ingests new member threads of an owned child after the root turn term
1404 const subagentNodeId = NodeId.make("node:run-execution-late-child:subagent");
1405 const childMessageIngested = yield* Deferred.make<void>();
1406 const order = yield* Ref.make<ReadonlyArray<string>>([]);
1407 + const origins = yield* Ref.make<ReadonlyArray<readonly [string, unknown]>>([]);
1408 const layerTest = RunExecutionService.layer.pipe(
1409 Layer.provide(
1410 Layer.mergeAll(
@@ -1424,6 +1430,11 @@ it.effect("ingests new member threads of an owned child after the root turn term
1430 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
1431 ingestNormalized: (input) =>
1432 Effect.gen(function* () {
1433 + const eventType = input.event.type;
1434 + yield* Ref.update(origins, (current) => [
1435 + ...current,
1436 + [eventType, input.analyticsContext?.origin] as const,
1437 + ]);
1438 if (
1439 input.event.type === "app_thread.created" &&
1440 input.event.appThread.id === memberThreadId
@@ -1559,7 +1570,7 @@ it.effect("ingests new member threads of an owned child after the root turn term
1570 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
1571 yield* runExecution.startRootRun({
1572 commandId: CommandId.make("command:run-execution-late-child"),
1562 - appThread: { id: threadId } as OrchestrationV2AppThread,
1573 + appThread: rootAppThread(threadId),
1574 providerSessionId,
1575 session: {
1576 events: Stream.fromIterable(events),
@@ -1612,6 +1623,19 @@ it.effect("ingests new member threads of an owned child after the root turn term
1623 );
1624 assert.isTrue(Option.isSome(observed), "child message was not ingested after root terminal");
1625 assert.deepEqual(yield* Ref.get(order), ["root-finalized", "member-thread", "child-message"]);
1626 + // The run's message started the root; the parent agent started the child's turn.
1627 + const recorded = yield* Ref.get(origins);
1628 + assert.deepEqual(recorded[0], [
1629 + "app_thread.created",
1630 + { createdBy: "user", creationSource: "web", scheduled: false, subagent: false },
1631 + ]);
1632 + assert.deepEqual(
1633 + recorded.find(([type]) => type === "provider_turn.updated"),
1634 + [
1635 + "provider_turn.updated",
1636 + { createdBy: "agent", creationSource: "provider", scheduled: false, subagent: true },
1637 + ],
1638 + );
1639 }),
1640 );
1641
@@ -1704,7 +1728,7 @@ it.effect("ends a nested native subagent and its thread when the root run is int
1728 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
1729 yield* runExecution.startRootRun({
1730 commandId: CommandId.make(`command:${key}`),
1707 - appThread: { id: threadId } as OrchestrationV2AppThread,
1731 + appThread: rootAppThread(threadId),
1732 providerSessionId: ProviderSessionId.make(`session:${key}`),
1733 session: {
1734 events: Stream.fromIterable(events),
@@ -2031,7 +2055,7 @@ it.effect(
2055 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
2056 yield* runExecution.startRootRun({
2057 commandId: CommandId.make("command:subagent-interrupt-cascade"),
2034 - appThread: { id: ids.threadId } as OrchestrationV2AppThread,
2058 + appThread: rootAppThread(ids.threadId),
2059 providerSessionId: ProviderSessionId.make("session:subagent-interrupt-cascade"),
2060 session: {
2061 events: Stream.empty,
@@ -2816,7 +2840,7 @@ function captureRootRunTermination(input: {
2840 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
2841 const start = runExecution.startRootRun({
2842 commandId: CommandId.make(`command:${input.key}`),
2819 - appThread: { id: ids.threadId } as OrchestrationV2AppThread,
2843 + appThread: rootAppThread(ids.threadId),
2844 providerSessionId: ProviderSessionId.make(`session:${input.key}`),
2845 session: {
2846 events: Stream.empty,
@@ -3312,7 +3336,7 @@ function runBackgroundItemScenario(
3336 const runExecution = yield* RunExecutionService.RunExecutionServiceV2;
3337 yield* runExecution.startRootRun({
3338 commandId: CommandId.make(`command:${key}`),
3315 - appThread: { id: ids.threadId } as OrchestrationV2AppThread,
3339 + appThread: rootAppThread(ids.threadId),
3340 providerSessionId: ProviderSessionId.make(`session:${key}`),
3341 session: {
3342 events: Stream.empty,
apps/server/src/orchestration-v2/RunExecutionService.ts
+19
@@ -97,6 +97,14 @@ function isSettledSubagentStatus(status: OrchestrationV2Subagent["status"]): boo
97 return !isOrchestrationV2WorkActive(status);
98 }
99
100 +/** A provider-native subagent's turns arrive on its parent's run, but the parent agent started them. */
101 +const NATIVE_SUBAGENT_TURN_ORIGIN: ProviderEventIngestor.ProviderTurnOrigin = {
102 + createdBy: "agent",
103 + creationSource: "provider",
104 + scheduled: false,
105 + subagent: true,
106 +};
107 +
108 const backgroundCapableTurnItemTypes: ReadonlySet<OrchestrationV2TurnItem["type"]> = new Set([
109 "command_execution",
110 "dynamic_tool",
@@ -883,6 +891,12 @@ export const layer: Layer.Layer<
891
892 return RunExecutionServiceV2.of({
893 startRootRun: Effect.fn("RunExecutionService.startRootRun")(function* (input) {
894 + const rootTurnOrigin: ProviderEventIngestor.ProviderTurnOrigin = {
895 + createdBy: input.message.createdBy,
896 + creationSource: input.message.creationSource,
897 + scheduled: input.message.scheduledTaskId !== undefined,
898 + subagent: input.appThread.lineage.relationshipToParent === "subagent",
899 + };
900 // Startup failure and stream shutdown can report the same attempt.
901 const refreshAfterTurn = yield* Effect.cached(
902 finalizationObserver.refreshAfterTurn(input.appThread.projectId).pipe(
@@ -1375,6 +1389,11 @@ export const layer: Layer.Layer<
1389 modelSelection: input.modelSelection,
1390 runtimeMode: input.runtimePolicy.runtimeMode,
1391 interactionMode: input.runtimePolicy.interactionMode,
1392 + origin:
1393 + event.type === "provider_turn.updated" &&
1394 + event.providerTurn.providerThreadId !== input.providerThread.id
1395 + ? NATIVE_SUBAGENT_TURN_ORIGIN
1396 + : rootTurnOrigin,
1397 },
1398 providerSessionId: input.providerSessionId,
1399 providerInstanceId: input.run.providerInstanceId,
apps/server/src/orchestration-v2/ThreadLaunchService.test.ts
+67
@@ -1597,6 +1597,73 @@ it.effect("does not let a failing same-command caller strand a concurrent durabl
1597 }),
1598 );
1599
1600 +it.effect("reports one concurrent same-command launch as new and the rest as resumed", () =>
1601 + Effect.gen(function* () {
1602 + // Both launches read no receipt before either one claims the thread.
1603 + const arrived = yield* Ref.make(0);
1604 + const bothRead = yield* Deferred.make<void>();
1605 + const harness = makeHarness({
1606 + managedFolders: Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({
1607 + namedProjectsRoot: "/projects",
1608 + isScratchProject: () => Effect.succeed(false),
1609 + folderForThread: () =>
1610 + Ref.updateAndGet(arrived, (count) => count + 1).pipe(
1611 + Effect.tap((count) =>
1612 + count === 2 ? Deferred.succeed(bothRead, undefined) : Effect.void,
1613 + ),
1614 + Effect.andThen(Deferred.await(bothRead)),
1615 + Effect.as(Option.none()),
1616 + ),
1617 + }),
1618 + });
1619 + yield* Effect.gen(function* () {
1620 + const launches = yield* ThreadLaunch.ThreadLaunchService;
1621 + const input = launchInput({
1622 + command: "command:launch:concurrent-resumed",
1623 + thread: "thread:launch:concurrent-resumed",
1624 + });
1625 + const results = yield* Effect.all([launches.launch(input), launches.launch(input)], {
1626 + concurrency: "unbounded",
1627 + });
1628 + assert.deepEqual(results.map((result) => result.resumed).sort(), [false, true]);
1629 + }).pipe(Effect.provide(harness.layer));
1630 + }),
1631 +);
1632 +
1633 +it.effect(
1634 + "does not report a retry as resumed when the earlier attempt only claimed the thread",
1635 + () =>
1636 + Effect.gen(function* () {
1637 + const harness = makeHarness();
1638 + yield* Effect.gen(function* () {
1639 + const launches = yield* ThreadLaunch.ThreadLaunchService;
1640 + const threads = yield* ThreadManagement.ThreadManagementService;
1641 + const input = launchInput({
1642 + command: "command:launch:claimed-only",
1643 + thread: "thread:launch:claimed-only",
1644 + message: "Counted once",
1645 + });
1646 + yield* threads.dispatch({
1647 + type: "thread.create",
1648 + commandId: input.commandId,
1649 + threadId: input.threadId,
1650 + projectId: input.projectId,
1651 + title: input.title,
1652 + modelSelection: input.modelSelection,
1653 + runtimeMode: input.runtimeMode,
1654 + interactionMode: input.interactionMode,
1655 + branch: null,
1656 + worktreePath: null,
1657 + createdBy: input.createdBy,
1658 + creationSource: input.creationSource,
1659 + });
1660 + const retried = yield* launches.launch(input);
1661 + assert.isFalse(retried.resumed);
1662 + assert.isTrue((yield* launches.launch(input)).resumed);
1663 + }).pipe(Effect.provide(harness.layer));
1664 + }),
1665 +);
1666 +
1667 it.effect("schedules an accepted preparing message exactly once across concurrent retries", () =>
1668 Effect.gen(function* () {
1669 const setupEntered = yield* Deferred.make<void>();
apps/server/src/orchestration-v2/ThreadLaunchService.ts
+9 -1
@@ -114,6 +114,7 @@ export interface ThreadLaunchPreparationInput {
114 export interface ThreadLaunchResult {
115 readonly threadId: ThreadId;
116 readonly projection: OrchestrationV2ThreadProjection;
117 + /** The launch's last write (the initial message, else the thread claim) was accepted before this call. */
118 readonly resumed: boolean;
119 }
120
@@ -974,6 +975,8 @@ const make = Effect.gen(function* () {
975 creationSource: input.creationSource,
976 })
977 .pipe(Effect.mapError(mapError(input, "dispatch-message", threadId)));
978 + // A concurrent retry can read no receipt and still replay the dispatch.
979 + messageWasAlreadyAccepted ||= dispatched.replayed === true;
980 const runCreated = dispatched.storedEvents.find(
981 (stored) => stored.event.type === "run.created",
982 );
@@ -1034,7 +1037,12 @@ const make = Effect.gen(function* () {
1037 return {
1038 threadId,
1039 projection,
1037 - resumed: Option.isSome(launchReceipt) || messageWasAlreadyAccepted,
1040 + // An attempt that failed after claiming the thread recorded nothing, so
1041 + // only a repeat of the final write counts as resumed.
1042 + resumed:
1043 + input.initialMessage === undefined
1044 + ? Option.isSome(launchReceipt) || claimed.replayed === true
1045 + : messageWasAlreadyAccepted,
1046 };
1047 });
1048 },
apps/server/src/telemetry/FeatureUsage.test.ts
+52 -13
@@ -1,29 +1,68 @@
1 -import { ORCHESTRATION_V2_WS_METHODS, ThreadId, WS_METHODS } from "@t3tools/contracts";
1 +import {
2 + ORCHESTRATION_V2_WS_METHODS,
3 + OrchestrationV2Command,
4 + WS_METHODS,
5 +} from "@t3tools/contracts";
6 import { expect, it } from "@effect/vitest";
7 +import * as Schema from "effect/Schema";
8
4 -import { featureUsage } from "./FeatureUsage.ts";
9 +import { commandFeatureUsage, featureUsage } from "./FeatureUsage.ts";
10
6 -const dispatch = ORCHESTRATION_V2_WS_METHODS.dispatchCommand;
7 -const threadId = ThreadId.make("thread-private");
11 +const command = Schema.decodeUnknownSync(OrchestrationV2Command);
12 +const ids = { commandId: "command-private", threadId: "thread-private" };
13
14 it("ignores messages, housekeeping commands, and untracked RPCs", () => {
10 - expect(featureUsage(dispatch, { type: "message.dispatch", text: "secret prompt" })).toBe(
11 - undefined,
12 - );
13 - expect(featureUsage(dispatch, { type: "thread.visit", threadId })).toBe(undefined);
15 + expect(
16 + commandFeatureUsage(
17 + command({
18 + ...ids,
19 + type: "message.dispatch",
20 + messageId: "message-private",
21 + text: "secret prompt",
22 + attachments: [],
23 + createdBy: "user",
24 + creationSource: "web",
25 + dispatchMode: { type: "start_immediately" },
26 + }),
27 + ),
28 + ).toBe(undefined);
29 + expect(
30 + commandFeatureUsage(
31 + command({ ...ids, type: "thread.visit", visitedAt: "2026-10-24T00:00:00.000Z" }),
32 + ),
33 + ).toBe(undefined);
34 expect(featureUsage(WS_METHODS.terminalWrite, { data: "rm -rf" })).toBe(undefined);
35 });
36
17 -it("reads RPC variants from enums, never from paths or text", () => {
18 - expect(
19 - featureUsage(WS_METHODS.shellOpenInEditor, { cwd: "/Users/someone/secret", editor: "cursor" }),
20 - ).toEqual({ feature: "editor.open", variant: "cursor" });
37 +it("leaves thread launches to client.thread.started", () => {
38 expect(
39 featureUsage(ORCHESTRATION_V2_WS_METHODS.launchThread, {
40 title: "Private title",
41 workspaceStrategy: { type: "worktree", baseRef: "main", branch: "secret-branch" },
42 }),
26 - ).toEqual({ feature: "thread.launch", variant: "worktree" });
43 + ).toBe(undefined);
44 +});
45 +
46 +it("tells watching a pull request apart from unwatching it", () => {
47 + const watch = (watching: boolean) =>
48 + commandFeatureUsage(
49 + command({
50 + ...ids,
51 + type: "thread.pull-request.watch",
52 + host: "github.com",
53 + repository: "owner/private",
54 + number: 7,
55 + watching,
56 + }),
57 + );
58 + expect(watch(true)).toEqual({ feature: "thread.pull_request.watch", variant: "on" });
59 + expect(watch(false)).toEqual({ feature: "thread.pull_request.watch", variant: "off" });
60 +});
61 +
62 +it("reads RPC variants from enums, never from paths or text", () => {
63 + expect(
64 + featureUsage(WS_METHODS.shellOpenInEditor, { cwd: "/Users/someone/secret", editor: "cursor" }),
65 + ).toEqual({ feature: "editor.open", variant: "cursor" });
66 expect(featureUsage(WS_METHODS.terminalOpen, { cwd: "/Users/someone/secret" })).toEqual({
67 feature: "terminal.open",
68 });
apps/server/src/telemetry/FeatureUsage.ts
+23 -28
@@ -1,8 +1,4 @@
1 -import {
2 - ORCHESTRATION_V2_WS_METHODS,
3 - type OrchestrationV2Command,
4 - WS_METHODS,
5 -} from "@t3tools/contracts";
1 +import { type OrchestrationV2Command, WS_METHODS } from "@t3tools/contracts";
2
3 export interface FeatureUsage {
4 readonly feature: string;
@@ -21,11 +17,14 @@ const use = (feature: string, variant?: string): FeatureUsage =>
17 * User commands sent through `orchestration.dispatchCommand` that represent a
18 * feature choice. Housekeeping commands (visits, reorders, acknowledgements)
19 * and `message.dispatch`, which `client.turn.requested` already counts, are
24 - * left out.
20 + * left out. The dispatch handler records these, not the RPC middleware, so a
21 + * retried command id that changed nothing is not counted again.
22 */
26 -const COMMAND_FEATURES: Partial<
27 - Record<OrchestrationV2Command["type"], (command: Payload) => FeatureUsage>
28 -> = {
23 +const COMMAND_FEATURES: {
24 + readonly [Type in OrchestrationV2Command["type"]]?: (
25 + command: Extract<OrchestrationV2Command, { readonly type: Type }>,
26 + ) => FeatureUsage;
27 +} = {
28 "checkpoint.rollback": (command) => ({
29 feature: "checkpoint.rollback",
30 variant: command.restoreFiles === false ? "conversation_only" : "with_files",
@@ -38,11 +37,11 @@ const COMMAND_FEATURES: Partial<
37 "thread.pin": () => ({ feature: "thread.pin" }),
38 "thread.settle": () => ({ feature: "thread.settle" }),
39 "thread.pull-request.link": () => ({ feature: "thread.pull_request.link" }),
41 - "thread.pull-request.watch": () => ({ feature: "thread.pull_request.watch" }),
42 - "thread.runtime-mode.set": (command) =>
43 - use("thread.runtime_mode.set", literal(command, "runtimeMode")),
40 + "thread.pull-request.watch": (command) =>
41 + use("thread.pull_request.watch", command.watching ? "on" : "off"),
42 + "thread.runtime-mode.set": (command) => use("thread.runtime_mode.set", command.runtimeMode),
43 "thread.interaction-mode.set": (command) =>
45 - use("thread.interaction_mode.set", literal(command, "interactionMode")),
44 + use("thread.interaction_mode.set", command.interactionMode),
45 "run.interrupt": () => ({ feature: "run.interrupt" }),
46 "queued-message.promote-to-steer": () => ({ feature: "queue.steer" }),
47 "queued-run.edit": () => ({ feature: "queue.edit" }),
@@ -51,17 +50,9 @@ const COMMAND_FEATURES: Partial<
50 /**
51 * RPCs that represent a user starting a feature. Variants come only from
52 * closed enums in the contracts, so no path, id, or free text can reach analytics.
53 + * Thread launches are left out because `client.thread.started` counts them.
54 */
55 const METHOD_FEATURES: Readonly<Record<string, (payload: Payload) => FeatureUsage>> = {
56 - [ORCHESTRATION_V2_WS_METHODS.launchThread]: (payload) => {
57 - const strategy = payload.workspaceStrategy;
58 - return use(
59 - "thread.launch",
60 - typeof strategy === "object" && strategy !== null
61 - ? literal(strategy as Payload, "type")
62 - : undefined,
63 - );
64 - },
56 [WS_METHODS.terminalOpen]: () => ({ feature: "terminal.open" }),
57 [WS_METHODS.previewOpen]: () => ({ feature: "preview.open" }),
58 [WS_METHODS.deviceOpen]: () => ({ feature: "device.open" }),
@@ -81,13 +72,17 @@ const METHOD_FEATURES: Readonly<Record<string, (payload: Payload) => FeatureUsag
72 [WS_METHODS.serverUpsertKeybinding]: () => ({ feature: "keybinding.customize" }),
73 };
74
75 +/** The feature a dispatched command uses, or undefined when it is not a tracked feature. */
76 +export function commandFeatureUsage(command: OrchestrationV2Command): FeatureUsage | undefined {
77 + // TypeScript cannot pair a union's tag with its own entry, so widen once here.
78 + const mapping = COMMAND_FEATURES[command.type] as
79 + | ((command: OrchestrationV2Command) => FeatureUsage)
80 + | undefined;
81 + return mapping?.(command);
82 +}
83 +
84 /** The feature a WebSocket request uses, or undefined when it is not a tracked feature. */
85 export function featureUsage(method: string, payload: unknown): FeatureUsage | undefined {
86 if (typeof payload !== "object" || payload === null) return undefined;
87 - const input = payload as Payload;
88 - if (method === ORCHESTRATION_V2_WS_METHODS.dispatchCommand) {
89 - const type = literal(input, "type") as OrchestrationV2Command["type"] | undefined;
90 - return type === undefined ? undefined : COMMAND_FEATURES[type]?.(input);
91 - }
92 - return METHOD_FEATURES[method]?.(input);
87 + return METHOD_FEATURES[method]?.(payload as Payload);
88 }
apps/server/src/ws.ts
+32 -19
@@ -229,7 +229,7 @@ import * as ProcessResourceMonitor from "./diagnostics/ProcessResourceMonitor.ts
229 import * as ResourceTelemetry from "./resourceTelemetry/ResourceTelemetry.ts";
230 import * as HostResources from "./resourceTelemetry/HostResources.ts";
231 import * as AnalyticsService from "./telemetry/AnalyticsService.ts";
232 -import { featureUsage } from "./telemetry/FeatureUsage.ts";
232 +import { commandFeatureUsage, featureUsage } from "./telemetry/FeatureUsage.ts";
233 import type { TelemetryOwner } from "./telemetry/Identify.ts";
234 import * as UsageService from "./usage/UsageService.ts";
235 import * as TraceDiagnostics from "./diagnostics/TraceDiagnostics.ts";
@@ -1247,12 +1247,18 @@ export const layerWsRpc = (
1247 const analytics = yield* AnalyticsService.AnalyticsService;
1248 const originProps = clientAnalyticsProps;
1249 const recordClientCommandAnalytics = (command: OrchestrationV2Command) => {
1250 - switch (command.type) {
1251 - case "message.dispatch":
1252 - return analytics.record("client.turn.requested", originProps).pipe(Effect.ignore);
1253 - default:
1254 - return Effect.void;
1250 + if (command.type === "message.dispatch") {
1251 + return analytics.record("client.turn.requested", originProps).pipe(Effect.ignore);
1252 }
1253 + const usage = commandFeatureUsage(command);
1254 + return usage === undefined
1255 + ? Effect.void
1256 + : analytics
1257 + .record("feature.used", {
1258 + ...usage,
1259 + ...(originProps.surface === undefined ? {} : { surface: originProps.surface }),
1260 + })
1261 + .pipe(Effect.ignore);
1262 };
1263 const threadLaunch = yield* ThreadLaunchService.ThreadLaunchService;
1264 const commandOutputHub = yield* CommandOutputHub.CommandOutputHub;
@@ -1889,7 +1895,9 @@ export const layerWsRpc = (
1895 ).pipe(Effect.provide(intakeContext)),
1896 )
1897 .pipe(
1892 - Effect.tap(() => recordClientCommandAnalytics(command)),
1898 + Effect.tap((result) =>
1899 + result.replayed ? Effect.void : recordClientCommandAnalytics(command),
1900 + ),
1901 Effect.map((result) => ({ sequence: result.sequence })),
1902 Effect.mapError((cause) => {
1903 const detail = userFacingDispatchErrorMessage(cause);
@@ -2028,17 +2036,23 @@ export const layerWsRpc = (
2036 }).pipe(Effect.provide(intakeContext)),
2037 )
2038 .pipe(
2031 - Effect.tap(() =>
2032 - analytics
2033 - .record("client.thread.started", originProps)
2034 - .pipe(
2035 - Effect.andThen(
2036 - input.initialMessage === undefined
2037 - ? Effect.void
2038 - : analytics.record("client.turn.requested", originProps),
2039 - ),
2040 - Effect.ignore,
2041 - ),
2039 + // A resumed launch is a retry of one already counted.
2040 + Effect.tap((result) =>
2041 + result.resumed
2042 + ? Effect.void
2043 + : analytics
2044 + .record("client.thread.started", {
2045 + ...originProps,
2046 + workspaceStrategy: input.workspaceStrategy.type,
2047 + })
2048 + .pipe(
2049 + Effect.andThen(
2050 + input.initialMessage === undefined
2051 + ? Effect.void
2052 + : analytics.record("client.turn.requested", originProps),
2053 + ),
2054 + Effect.ignore,
2055 + ),
2056 ),
2057 Effect.map((result) => ({
2058 ...result,
@@ -3233,7 +3247,6 @@ export const layer = Layer.unwrap(
3247 const clientOrigin = readClientConnectionOrigin(request);
3248 const clientAnalyticsProps = readClientAnalyticsProps(request);
3249 yield* sessions.recordClientConnection(session.sessionId, clientOrigin);
3236 - yield* analytics.record("client.connected", clientAnalyticsProps);
3250 yield* analytics.recordOwners(readClientOwners(request), "client");
3251 const rpcWebSocketHttpEffect = yield* Effect.gen(function* () {
3252 const { protocol, httpEffect } = yield* RpcServer.makeProtocolWithHttpEffectWebsocket;
docs/internals/product-analytics.md
+21 -10
@@ -47,13 +47,17 @@ runs; use `surface` for the connected client.
47
48 ## Interpreting events
49
50 -Use `client.turn.requested` for active-use reports. `client.connected` counts
51 -reconnects, so network behavior can inflate it. One identity can appear in several
52 -client groups during a period; adding those groups double-counts users.
53 -
54 -Provider send and completion counts need not match. Providers can emit synthetic
55 -turns without a send request. Collection is best effort, with no scan or backfill
56 -of provider history.
50 +Use `client.turn.requested` for active-use reports. There is no connection event,
51 +because T3 Connect sessions expire hourly and each reconnect would count again.
52 +A command whose id was already accepted is a retry, so it records nothing again.
53 +One identity can appear in several client groups during a period; adding those
54 +groups double-counts users.
55 +
56 +Provider send and completion counts need not match. Most completions start
57 +without a person: provider continuations, agent sends, and delegated subagents.
58 +`provider.turn.completed` carries the starting message's `createdBy` and
59 +`creationSource`, plus `scheduled` and `subagent`; filter on them to count user
60 +turns. Collection is best effort, with no scan or backfill of provider history.
61
62 Token totals cover the main agent. Child agents and model rerouting prevent a turn
63 from representing one provider/model combination's full cost. For provider
@@ -66,8 +70,9 @@ Unknown counts stay absent. Partial usage contains valid observed counts but
70 cannot establish a whole-turn total. Keep these distinctions when changing token
71 normalization or building reports.
72
69 -`feature.used` counts WebSocket requests that start a feature, observed in the
70 -RPC authorization middleware after the request succeeds.
73 +`feature.used` counts WebSocket requests that start a feature. RPCs are observed in
74 +the RPC authorization middleware after the request succeeds; dispatched commands
75 +are recorded by the dispatch handler, which can tell a retry apart.
76 [Feature mapping](../../apps/server/src/telemetry/FeatureUsage.ts) allowlists
77 the methods and commands, and takes variants only from closed contract enums,
78 so paths, ids, and text never reach analytics.
@@ -80,7 +85,9 @@ Pi, and ACP agents list models from user configuration, such as local Ollama
85 tags, so their models stay out. Handoff settings come from closed enums and
86 booleans in tool arguments; prompts, titles, and ids are never read. Thread ids
87 are used only to look up the threads on each side and never leave the server,
83 -which is also why tool events carry no per-thread sequence.
88 +which is also why tool events carry no per-thread sequence. A handoff to several
89 +threads, such as `create_threads`, records one `mcp.tool.invoked` per target, each
90 +repeating the call's outcome and duration, so event counts are not call counts.
91
92 ## Delivery
93
@@ -98,3 +105,7 @@ Keep analytics payloads to product metadata and normalized measurements. Do not
105 send prompts, authentication material, raw provider payloads, user-assigned device
106 names, or conversation identifiers. Client metadata is best effort; invalid values
107 must not reject a connection. PostHog person profiles remain disabled.
108 +
109 +Dev-runner servers and the CI archive smoke test turn telemetry off, which also
110 +stops relay tracing, because each would count as a new install. Set
111 +`T3CODE_TELEMETRY_ENABLED=true` to test analytics or relay tracing from a dev server.
scripts/dev-runner.test.ts
+25
@@ -90,6 +90,31 @@ it.layer(NodeServices.layer)("dev-runner", (it) => {
90 }),
91 );
92
93 + it.effect("turns product analytics off unless the developer opts in", () =>
94 + Effect.gen(function* () {
95 + const input = {
96 + mode: "dev",
97 + serverOffset: 0,
98 + webOffset: 0,
99 + t3Home: undefined,
100 + browser: undefined,
101 + autoBootstrapProjectFromCwd: undefined,
102 + logWebSocketEvents: undefined,
103 + host: undefined,
104 + port: undefined,
105 + devUrl: undefined,
106 + } as const;
107 + const byDefault = yield* createDevRunnerEnv({ ...input, baseEnv: {} });
108 + const optedIn = yield* createDevRunnerEnv({
109 + ...input,
110 + baseEnv: { T3CODE_TELEMETRY_ENABLED: "true" },
111 + });
112 +
113 + assert.equal(byDefault.T3CODE_TELEMETRY_ENABLED, "false");
114 + assert.equal(optedIn.T3CODE_TELEMETRY_ENABLED, "true");
115 + }),
116 + );
117 +
118 it.effect("strips inherited service-launcher context", () =>
119 Effect.gen(function* () {
120 const env = yield* createDevRunnerEnv({
scripts/dev-runner.ts
+4
@@ -344,6 +344,10 @@ export const createDevRunnerEnv = Effect.fn("dev-runner.createDevRunnerEnv")(fun
344 delete output.T3_SERVICE_LAUNCHER_CONTEXT;
345 delete output.T3_BOOT_SERVICE_UNIT;
346
347 + // Each worktree has its own state, so a dev server would count as a new
348 + // install of the developer's real accounts. Set it explicitly to opt in.
349 + output.T3CODE_TELEMETRY_ENABLED = baseEnv.T3CODE_TELEMETRY_ENABLED ?? "false";
350 +
351 if (!isDesktopMode) {
352 output.T3CODE_PORT = String(serverPort);
353 // HOST is Vite's own bind address, and the desktop branch below is the
scripts/smoke-cli-archive.ts
+3
@@ -134,6 +134,9 @@ const smokeCliArchive = Effect.fn("smokeCliArchive")(function* (input: {
134 TMPDIR: scratch,
135 TEMP: scratch,
136 T3CODE_HOME: home,
137 + // The archive has a release channel built in, so its events would
138 + // look like a real new install.
139 + T3CODE_TELEMETRY_ENABLED: "false",
140 },
141 extendEnv: false,
142 },