The best way to control Claude Code, Codex and more from any computer or phone
feat(server): export how long runs take and how they end (#22094)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Julius Marminge committed
Oct 21, 2026 at 18:49 UTC
f8e4834861d255f58c44fc1b5f57c34c49f5384d
4 files changed
+454
-20
apps/server/src/observability/Metrics.ts
+12
@@ -63,6 +63,18 @@ export const serverEventLoopDelayMax = Metric.gauge("t3_server_event_loop_delay_
63
attributes: { unit: "ms" },
64
});
65
66
+export const runsTotal = Metric.counter("t3_runs_total", {
67
+ description: "Runs that finished, by outcome. A later rollback does not count again.",
68
+});
69
+
70
+export const runDuration = Metric.timer("t3_run_duration", {
71
+ description: "Wall time of a run, from when it started to its terminal status.",
72
+});
73
+
74
+export const runQueueWait = Metric.timer("t3_run_queue_wait", {
75
+ description: "Time a run waited between being requested and starting.",
76
+});
77
+
78
export const gitCommandsTotal = Metric.counter("t3_git_commands_total", {
79
description: "Total git commands executed by the server runtime.",
80
});
apps/server/src/orchestration-v2/EventSink.ts
+45
-20
@@ -26,6 +26,7 @@ import * as SqlClient from "effect/sql/SqlClient";
26
import { replayAndBufferProjectedLiveEvents } from "./LiveStreamBudget.ts";
27
import type { UnsequencedProjectEvent } from "../persistence/OrchestrationEventStore.ts";
28
import { projectDomainEventForWire } from "./WireProjection.ts";
29
+import { makeRunOutcomeRecorder, type RunOutcome } from "./RunMetrics.ts";
30
31
import * as CommandReceiptStore from "./CommandReceiptStore.ts";
32
import * as EffectOutbox from "./EffectOutbox.ts";
@@ -243,8 +244,23 @@ const layerBase: Layer.Layer<
244
);
245
}
246
});
246
- const publishStoredEvents = (events: ReadonlyArray<OrchestrationV2StoredEvent>) =>
247
- eventStore.publishCommitted(events).pipe(Effect.andThen(publishLiveEvents(events)));
247
+ const runOutcomes = makeRunOutcomeRecorder((threadId) =>
248
+ // One payload row; unlike the shell, it still resolves a deleted child thread.
249
+ projectionStore
250
+ .getThread(threadId)
251
+ .pipe(Effect.map((thread) => thread.lineage.relationshipToParent === "subagent")),
252
+ );
253
+ // Recorded before publication, so an interrupted publish cannot lose a committed outcome.
254
+ const publishStoredEvents = (
255
+ events: ReadonlyArray<OrchestrationV2StoredEvent>,
256
+ outcomes: ReadonlyArray<RunOutcome>,
257
+ ) =>
258
+ runOutcomes
259
+ .record(outcomes)
260
+ .pipe(
261
+ Effect.andThen(eventStore.publishCommitted(events)),
262
+ Effect.andThen(publishLiveEvents(events)),
263
+ );
264
265
// Transactions commit one at a time, but each writer publishes after its
266
// commit. If a writer is descheduled in between, a later commit reaches
@@ -373,6 +389,8 @@ const layerBase: Layer.Layer<
389
updated_at = excluded.updated_at
390
`;
391
}
392
+ // Read lineage here, on the connection this transaction holds; see makeRunOutcomeRecorder.
393
+ return yield* runOutcomes.prepare(storedEvents);
394
});
395
396
const writeEffect = Effect.fn("orchestrationV2.EventSink.write")(function* (
@@ -384,7 +402,7 @@ const layerBase: Layer.Layer<
402
"orchestration_v2.thread_id": input.events[0]?.threadId ?? null,
403
});
404
387
- return yield* commitThenPublish(
405
+ const { committed } = yield* commitThenPublish(
406
Effect.gen(function* () {
407
// Check ownership in the same transaction that claims the native session.
408
const imported = input.unlessNativeThreadOwned;
@@ -398,7 +416,7 @@ const layerBase: Layer.Layer<
416
AND json_extract(payload_json, '$.nativeThreadRef.nativeId') = ${imported.nativeThreadRef.nativeId}
417
LIMIT 1
418
`;
401
- if (owners.length > 0) return [];
419
+ if (owners.length > 0) return { committed: [], outcomes: [] };
420
}
421
// Checkpoint capture can finish after a cohort writer reads its parent run.
422
// Merge in this transaction so the event log and projection keep that completion.
@@ -433,18 +451,19 @@ const layerBase: Layer.Layer<
451
...(input.commandId === undefined ? {} : { commandId: input.commandId }),
452
events: normalized,
453
});
436
- yield* applyStoredEvents(committed);
454
+ const outcomes = yield* applyStoredEvents(committed);
455
yield* effectOutbox.enqueue(input.effects);
438
- return committed;
456
+ return { committed, outcomes };
457
}).pipe(Effect.withTracerEnabled(false)),
440
- (storedEvents) =>
458
+ ({ committed, outcomes }) =>
459
Effect.gen(function* () {
460
if (input.effects.length > 0) {
461
yield* effectOutbox.notifyAvailable(input.effects.length);
462
}
445
- yield* publishStoredEvents(storedEvents);
463
+ yield* publishStoredEvents(committed, outcomes);
464
}),
465
);
466
+ return committed;
467
});
468
469
const writeIfRunCurrentEffect = Effect.fn("orchestrationV2.EventSink.writeIfRunCurrent")(
@@ -491,9 +510,9 @@ const layerBase: Layer.Layer<
510
...(input.commandId === undefined ? {} : { commandId: input.commandId }),
511
events: normalized,
512
});
494
- yield* applyStoredEvents(storedEvents);
513
+ const outcomes = yield* applyStoredEvents(storedEvents);
514
yield* effectOutbox.enqueue(input.effects ?? []);
496
- return { committed: true as const, storedEvents };
515
+ return { committed: true as const, storedEvents, outcomes };
516
}),
517
(result) =>
518
Effect.gen(function* () {
@@ -501,7 +520,7 @@ const layerBase: Layer.Layer<
520
if (input.effects !== undefined && input.effects.length > 0) {
521
yield* effectOutbox.notifyAvailable(input.effects.length);
522
}
504
- yield* publishStoredEvents(result.storedEvents);
523
+ yield* publishStoredEvents(result.storedEvents, result.outcomes);
524
}),
525
);
526
},
@@ -556,10 +575,13 @@ const layerBase: Layer.Layer<
575
...(input.commandId === undefined ? {} : { commandId: input.commandId }),
576
events: normalized,
577
});
559
- yield* applyStoredEvents(storedEvents);
560
- return { committed: true as const, storedEvents };
578
+ const outcomes = yield* applyStoredEvents(storedEvents);
579
+ return { committed: true as const, storedEvents, outcomes };
580
}),
562
- (result) => (result.committed ? publishStoredEvents(result.storedEvents) : Effect.void),
581
+ (result) =>
582
+ result.committed
583
+ ? publishStoredEvents(result.storedEvents, result.outcomes)
584
+ : Effect.void,
585
);
586
});
587
@@ -589,10 +611,13 @@ const layerBase: Layer.Layer<
611
612
const normalized = yield* normalizeEvents(input.events);
613
const storedEvents = yield* eventStore.append({ events: normalized });
592
- yield* applyStoredEvents(storedEvents);
593
- return { committed: true as const, storedEvents };
614
+ const outcomes = yield* applyStoredEvents(storedEvents);
615
+ return { committed: true as const, storedEvents, outcomes };
616
}),
595
- (result) => (result.committed ? publishStoredEvents(result.storedEvents) : Effect.void),
617
+ (result) =>
618
+ result.committed
619
+ ? publishStoredEvents(result.storedEvents, result.outcomes)
620
+ : Effect.void,
621
);
622
});
623
@@ -642,7 +667,7 @@ const layerBase: Layer.Layer<
667
new Error(`Command ${input.commandId} produced no orchestration events.`),
668
);
669
}
645
- yield* applyStoredEvents(storedEvents);
670
+ const outcomes = yield* applyStoredEvents(storedEvents);
671
yield* effectOutbox.enqueue(input.effects);
672
const receipt: CommandReceiptStore.CommandReceiptV2 = {
673
commandId: input.commandId,
@@ -661,7 +686,7 @@ const layerBase: Layer.Layer<
686
threadId: input.threadId,
687
...input.cancelUnsettledEffects,
688
});
664
- return { receipt, storedEvents, committed: true as const, cancelledEffectIds };
689
+ return { receipt, storedEvents, outcomes, committed: true as const, cancelledEffectIds };
690
}),
691
(result) =>
692
Effect.gen(function* () {
@@ -669,7 +694,7 @@ const layerBase: Layer.Layer<
694
if (result.committed && input.effects.length > 0) {
695
yield* effectOutbox.notifyAvailable(input.effects.length);
696
}
672
- if (result.committed) yield* publishStoredEvents(result.storedEvents);
697
+ if (result.committed) yield* publishStoredEvents(result.storedEvents, result.outcomes);
698
}),
699
);
700
return {
apps/server/src/orchestration-v2/RunMetrics.test.ts
new
+284
@@ -0,0 +1,284 @@
1
+import { assert, describe, it } from "@effect/vitest";
2
+import {
3
+ EventId,
4
+ MessageId,
5
+ ProjectId,
6
+ ProviderInstanceId,
7
+ RunId,
8
+ ThreadId,
9
+ type OrchestrationV2AppThread,
10
+ type OrchestrationV2DomainEvent,
11
+ type OrchestrationV2Run,
12
+ type OrchestrationV2StoredEvent,
13
+} from "@t3tools/contracts";
14
+import * as DateTime from "effect/DateTime";
15
+import * as Deferred from "effect/Deferred";
16
+import * as Effect from "effect/Effect";
17
+import * as Fiber from "effect/Fiber";
18
+import * as Layer from "effect/Layer";
19
+import * as Metric from "effect/Metric";
20
+import * as Option from "effect/Option";
21
+import * as SqlClient from "effect/sql/SqlClient";
22
+
23
+import * as SqlitePersistence from "../persistence/Sqlite.ts";
24
+import * as EventSink from "./EventSink.ts";
25
+import * as EventStore from "./EventStore.ts";
26
+import * as ProjectionStore from "./ProjectionStore.ts";
27
+import { makeRunOutcomeRecorder } from "./RunMetrics.ts";
28
+
29
+const at = (iso: string) => DateTime.makeUnsafe(iso);
30
+const threadId = ThreadId.make("thread:metrics");
31
+
32
+// Each test labels its runs with its own provider, since the metric registry is shared.
33
+const runUpdated = (
34
+ sequence: number,
35
+ provider: string,
36
+ run: Pick<OrchestrationV2Run, "id" | "status" | "startedAt" | "completedAt"> &
37
+ Partial<OrchestrationV2Run>,
38
+): OrchestrationV2StoredEvent => ({
39
+ sequence,
40
+ commandId: null,
41
+ event: {
42
+ type: "run.updated",
43
+ id: EventId.make(`event:${provider}:${sequence}`),
44
+ threadId,
45
+ occurredAt: at("2026-10-19T12:00:00.000Z"),
46
+ payload: {
47
+ threadId,
48
+ ordinal: 1,
49
+ providerInstanceId: ProviderInstanceId.make(provider),
50
+ modelSelection: { instanceId: ProviderInstanceId.make(provider), model: "claude-opus-5-5" },
51
+ requestedAt: at("2026-10-19T12:00:00.000Z"),
52
+ ...run,
53
+ } as OrchestrationV2Run,
54
+ },
55
+});
56
+
57
+// Prepare inside the transaction, record after its commit, as the EventSink does.
58
+const commitWith =
59
+ (recorder: ReturnType<typeof makeRunOutcomeRecorder<unknown>>) =>
60
+ (events: ReadonlyArray<OrchestrationV2StoredEvent>) =>
61
+ recorder.prepare(events).pipe(Effect.flatMap(recorder.record));
62
+
63
+const runSnapshots = (provider: string) =>
64
+ Metric.snapshot.pipe(
65
+ Effect.map((snapshots) =>
66
+ snapshots.filter(
67
+ (snapshot) =>
68
+ snapshot.id.startsWith("t3_run") && snapshot.attributes?.["provider"] === provider,
69
+ ),
70
+ ),
71
+ );
72
+
73
+describe("makeRunOutcomeRecorder", () => {
74
+ it.effect("records a finished run once, labelled, with its queue wait and duration", () =>
75
+ Effect.gen(function* () {
76
+ // A subagent thread's first run carries no delegatedTaskId; only its lineage says delegated.
77
+ const recorder = makeRunOutcomeRecorder(() => Effect.succeed(true));
78
+ const record = commitWith(recorder);
79
+ const finished = {
80
+ id: RunId.make("run:metrics:once"),
81
+ status: "completed" as const,
82
+ startedAt: at("2026-10-19T12:00:30.000Z"),
83
+ completedAt: at("2026-10-19T12:10:30.000Z"),
84
+ };
85
+
86
+ yield* record([runUpdated(1, "p1", { ...finished, status: "running", completedAt: null })]);
87
+ // A transaction that rolls back prepares and never records; it must not reserve the run.
88
+ yield* recorder.prepare([runUpdated(2, "p1", finished)]);
89
+ yield* record([runUpdated(2, "p1", finished)]);
90
+ // Later updates to a finished run, such as its checkpoint, are not another run.
91
+ yield* record([runUpdated(3, "p1", finished)]);
92
+
93
+ const snapshots = yield* runSnapshots("p1");
94
+ const byId = (id: string) => snapshots.find((snapshot) => snapshot.id === id);
95
+ const counter = byId("t3_runs_total");
96
+ assert.strictEqual(counter?.type === "Counter" ? counter.state.count : undefined, 1);
97
+ assert.deepStrictEqual(counter?.attributes, {
98
+ provider: "p1",
99
+ modelFamily: "claude",
100
+ outcome: "completed",
101
+ delegated: "true",
102
+ });
103
+ const histogramSum = (id: string) => {
104
+ const snapshot = byId(id);
105
+ return snapshot?.type === "Histogram" ? snapshot.state.sum : undefined;
106
+ };
107
+ assert.strictEqual(histogramSum("t3_run_queue_wait"), 30_000);
108
+ assert.strictEqual(histogramSum("t3_run_duration"), 600_000);
109
+ }),
110
+ );
111
+
112
+ it.effect("does not count a rollback of a run that already finished", () =>
113
+ Effect.gen(function* () {
114
+ // A fresh recorder stands in for a restart, which forgets the runs already counted.
115
+ const record = commitWith(makeRunOutcomeRecorder(() => Effect.succeed(false)));
116
+ yield* record([
117
+ runUpdated(1, "p2", {
118
+ id: RunId.make("run:metrics:rolled-back"),
119
+ status: "rolled_back",
120
+ startedAt: at("2026-10-19T12:00:30.000Z"),
121
+ completedAt: at("2026-10-20T09:00:00.000Z"),
122
+ }),
123
+ ]);
124
+
125
+ assert.deepStrictEqual(yield* runSnapshots("p2"), []);
126
+ }),
127
+ );
128
+});
129
+
130
+describe("EventSink run outcomes", () => {
131
+ const parentId = ThreadId.make("thread:metrics:sink:parent");
132
+ const childId = ThreadId.make("thread:metrics:sink:child");
133
+ const provider = ProviderInstanceId.make("p-sink");
134
+ const now = at("2026-10-19T12:00:00.000Z");
135
+ const thread = (id: ThreadId, lineage: OrchestrationV2AppThread["lineage"]) =>
136
+ ({
137
+ type: "thread.created",
138
+ id: EventId.make(`event:${id}:created`),
139
+ threadId: id,
140
+ occurredAt: now,
141
+ payload: {
142
+ createdBy: "user",
143
+ creationSource: "web",
144
+ id,
145
+ projectId: ProjectId.make("project:metrics"),
146
+ title: id,
147
+ providerInstanceId: provider,
148
+ modelSelection: { instanceId: provider, model: "gpt-6.1" },
149
+ runtimeMode: "full-access",
150
+ interactionMode: "default",
151
+ branch: null,
152
+ worktreePath: null,
153
+ activeProviderThreadId: null,
154
+ lineage,
155
+ forkedFrom: null,
156
+ createdAt: now,
157
+ updatedAt: now,
158
+ archivedAt: null,
159
+ settledOverride: null,
160
+ settledAt: null,
161
+ lastVisitedAt: null,
162
+ deletedAt: null,
163
+ },
164
+ }) satisfies OrchestrationV2DomainEvent;
165
+ const runId = RunId.make("run:metrics:sink:child");
166
+ const run = (status: OrchestrationV2Run["status"]): OrchestrationV2Run => ({
167
+ id: runId,
168
+ threadId: childId,
169
+ ordinal: 1,
170
+ providerInstanceId: provider,
171
+ modelSelection: { instanceId: provider, model: "gpt-6.1" },
172
+ providerThreadId: null,
173
+ userMessageId: MessageId.make("message:metrics:sink"),
174
+ rootNodeId: null,
175
+ activeAttemptId: null,
176
+ status,
177
+ queuePosition: null,
178
+ requestedAt: now,
179
+ startedAt: at("2026-10-19T12:00:10.000Z"),
180
+ completedAt: status === "running" ? null : at("2026-10-19T12:01:10.000Z"),
181
+ checkpointId: null,
182
+ contextHandoffId: null,
183
+ });
184
+ const runEvent = (type: "run.created" | "run.updated", status: OrchestrationV2Run["status"]) =>
185
+ ({
186
+ type,
187
+ id: EventId.make(`event:${runId}:${status}`),
188
+ threadId: childId,
189
+ runId,
190
+ occurredAt: now,
191
+ payload: run(status),
192
+ }) satisfies OrchestrationV2DomainEvent;
193
+
194
+ it.effect("reads lineage inside the transaction, so a concurrent writer cannot deadlock it", () =>
195
+ Effect.gen(function* () {
196
+ const lookupEntered = yield* Deferred.make<boolean>();
197
+ const releaseLookup = yield* Deferred.make<void>();
198
+ const writerStarted = yield* Deferred.make<void>();
199
+ const pausedLookup = Layer.effect(
200
+ ProjectionStore.ProjectionStoreV2,
201
+ Effect.gen(function* () {
202
+ const store = yield* ProjectionStore.ProjectionStoreV2;
203
+ const sql = yield* SqlClient.SqlClient;
204
+ let paused = false;
205
+ return {
206
+ ...store,
207
+ getThread: (id) =>
208
+ Effect.gen(function* () {
209
+ if (id === childId && !paused) {
210
+ paused = true;
211
+ const inTransaction = Option.isSome(
212
+ yield* Effect.serviceOption(sql.transactionService),
213
+ );
214
+ yield* Deferred.succeed(lookupEntered, inTransaction);
215
+ // Outside the transaction this read would wait on the writer it is racing.
216
+ if (inTransaction) yield* Deferred.await(releaseLookup);
217
+ }
218
+ return yield* store.getThread(id);
219
+ }),
220
+ };
221
+ }),
222
+ );
223
+ const layer = EventSink.layer.pipe(
224
+ Layer.provideMerge(pausedLookup),
225
+ Layer.provideMerge(
226
+ Layer.mergeAll(EventStore.layer, ProjectionStore.layer).pipe(
227
+ Layer.provideMerge(SqlitePersistence.layerMemory),
228
+ ),
229
+ ),
230
+ );
231
+
232
+ yield* Effect.gen(function* () {
233
+ const sink = yield* EventSink.EventSinkV2;
234
+ yield* sink.write({
235
+ events: [
236
+ thread(parentId, {
237
+ parentThreadId: null,
238
+ relationshipToParent: null,
239
+ rootThreadId: parentId,
240
+ }),
241
+ thread(childId, {
242
+ parentThreadId: parentId,
243
+ relationshipToParent: "subagent",
244
+ rootThreadId: parentId,
245
+ }),
246
+ runEvent("run.created", "running"),
247
+ ],
248
+ });
249
+ const terminal = yield* sink
250
+ .write({ events: [runEvent("run.updated", "completed")] })
251
+ .pipe(Effect.forkScoped);
252
+ assert.isTrue(yield* Deferred.await(lookupEntered), "lineage read outside the transaction");
253
+ const unrelated = yield* Deferred.succeed(writerStarted, undefined).pipe(
254
+ Effect.andThen(
255
+ sink.write({
256
+ events: [
257
+ thread(ThreadId.make("thread:metrics:sink:other"), {
258
+ parentThreadId: null,
259
+ relationshipToParent: null,
260
+ rootThreadId: ThreadId.make("thread:metrics:sink:other"),
261
+ }),
262
+ ],
263
+ }),
264
+ ),
265
+ Effect.forkScoped,
266
+ );
267
+ yield* Deferred.await(writerStarted);
268
+ yield* Deferred.succeed(releaseLookup, undefined);
269
+
270
+ const [terminalEvents, unrelatedEvents] = [
271
+ yield* Fiber.join(terminal),
272
+ yield* Fiber.join(unrelated),
273
+ ];
274
+ assert.strictEqual(terminalEvents.length, 1);
275
+ assert.strictEqual(unrelatedEvents.length, 1);
276
+ const counter = (yield* runSnapshots("p-sink")).find(
277
+ (snapshot) => snapshot.id === "t3_runs_total",
278
+ );
279
+ assert.strictEqual(counter?.type === "Counter" ? counter.state.count : undefined, 1);
280
+ assert.strictEqual(counter?.attributes?.["delegated"], "true");
281
+ }).pipe(Effect.provide(layer));
282
+ }),
283
+ );
284
+});
apps/server/src/orchestration-v2/RunMetrics.ts
new
+113
@@ -0,0 +1,113 @@
1
+import type {
2
+ OrchestrationV2Run,
3
+ OrchestrationV2StoredEvent,
4
+ RunId,
5
+ ThreadId,
6
+} from "@t3tools/contracts";
7
+import * as DateTime from "effect/DateTime";
8
+import type * as Duration from "effect/Duration";
9
+import * as Effect from "effect/Effect";
10
+import * as Metric from "effect/Metric";
11
+
12
+import { normalizeModelMetricLabel } from "../observability/Attributes.ts";
13
+import {
14
+ metricAttributes,
15
+ runDuration,
16
+ runQueueWait,
17
+ runsTotal,
18
+} from "../observability/Metrics.ts";
19
+
20
+const REMEMBERED_RUNS = 4096;
21
+// A rollback rewrites runs that already finished, stamping the rollback time as their completion,
22
+// so it is not counted as an outcome. Declared here because ThreadManagementService imports
23
+// EventSink.
24
+const OUTCOMES: ReadonlySet<OrchestrationV2Run["status"]> = new Set([
25
+ "completed",
26
+ "failed",
27
+ "cancelled",
28
+ "interrupted",
29
+]);
30
+
31
+export interface RunOutcome {
32
+ readonly runId: RunId;
33
+ readonly attributes: ReturnType<typeof metricAttributes>;
34
+ readonly queueWait: Duration.Duration | undefined;
35
+ readonly duration: Duration.Duration | undefined;
36
+}
37
+
38
+/**
39
+ * Records each run once, when a committed event first shows it finished. A finished run's row is
40
+ * updated again later (its checkpoint, a settlement), so recently counted runs are remembered.
41
+ *
42
+ * A run carries `delegatedTaskId` only when a follow-up is associated with a task, so a subagent
43
+ * thread's first run is recognized by its thread's lineage instead. A failed lookup only costs the
44
+ * label: the events still commit and publish.
45
+ *
46
+ * `prepare` reads lineage, so the EventSink runs it inside the write transaction, on the
47
+ * connection that transaction already holds. `record` runs after the commit, under the publish
48
+ * lane, and must stay free of I/O: a read there waits for the writer connection while holding the
49
+ * lane, and the next writer holds that connection while it waits for the lane.
50
+ */
51
+export const makeRunOutcomeRecorder = <E>(
52
+ isSubagentThread: (threadId: ThreadId) => Effect.Effect<boolean, E>,
53
+) => {
54
+ const counted = new Set<string>();
55
+ // Untraced: both run for every committed batch inside the EventSink operation's span.
56
+ // Never reads or writes `counted`, so a transaction that rolls back reserves nothing.
57
+ const prepare = Effect.fnUntraced(function* (events: ReadonlyArray<OrchestrationV2StoredEvent>) {
58
+ const outcomes: Array<RunOutcome> = [];
59
+ // Every finished run is prepared, in order, even one already counted: an earlier outcome in
60
+ // this batch may evict it before `record` reaches it, so only `record` decides what counts.
61
+ // Only the lineage read is shared.
62
+ const delegatedThreads = new Map<ThreadId, boolean>();
63
+ for (const { event } of events) {
64
+ if (event.type !== "run.updated") continue;
65
+ const run = event.payload;
66
+ if (!OUTCOMES.has(run.status)) continue;
67
+ let delegated = run.delegatedTaskId !== undefined || delegatedThreads.get(run.threadId);
68
+ if (delegated === undefined) {
69
+ delegated = yield* isSubagentThread(run.threadId).pipe(Effect.orElseSucceed(() => false));
70
+ delegatedThreads.set(run.threadId, delegated);
71
+ }
72
+ outcomes.push({
73
+ runId: run.id,
74
+ attributes: metricAttributes({
75
+ provider: run.providerInstanceId,
76
+ modelFamily: normalizeModelMetricLabel(run.modelSelection.model),
77
+ outcome: run.status,
78
+ delegated,
79
+ }),
80
+ queueWait:
81
+ run.startedAt === null ? undefined : DateTime.distance(run.requestedAt, run.startedAt),
82
+ duration:
83
+ run.startedAt === null || run.completedAt === null
84
+ ? undefined
85
+ : DateTime.distance(run.startedAt, run.completedAt),
86
+ });
87
+ }
88
+ return outcomes;
89
+ });
90
+ // Uninterruptible so a run is never marked counted with only part of its measurements.
91
+ const record = Effect.fnUntraced(function* (outcomes: ReadonlyArray<RunOutcome>) {
92
+ for (const outcome of outcomes) {
93
+ // Concurrent writers can prepare the same run; the first to publish counts it.
94
+ if (counted.has(outcome.runId)) continue;
95
+ counted.add(outcome.runId);
96
+ if (counted.size > REMEMBERED_RUNS) counted.delete(counted.values().next().value!);
97
+ yield* Metric.update(Metric.withAttributes(runsTotal, outcome.attributes), 1);
98
+ if (outcome.queueWait !== undefined) {
99
+ yield* Metric.update(
100
+ Metric.withAttributes(runQueueWait, outcome.attributes),
101
+ outcome.queueWait,
102
+ );
103
+ }
104
+ if (outcome.duration !== undefined) {
105
+ yield* Metric.update(
106
+ Metric.withAttributes(runDuration, outcome.attributes),
107
+ outcome.duration,
108
+ );
109
+ }
110
+ }
111
+ }, Effect.uninterruptible);
112
+ return { prepare, record };
113
+};