@devries-cameron20 / t3code Mirror · read-only / commits / f8e4834861
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 19:30 UTC

feat(server): export how long runs take and how they end (#22094)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Julius Marminge committed Oct 21, 2026 at 18:49 UTC f8e4834861d255f58c44fc1b5f57c34c49f5384d
4 files changed +454 -20
apps/server/src/observability/Metrics.ts
+12
@@ -63,6 +63,18 @@ export const serverEventLoopDelayMax = Metric.gauge("t3_server_event_loop_delay_
63 attributes: { unit: "ms" },
64 });
65
66 +export const runsTotal = Metric.counter("t3_runs_total", {
67 + description: "Runs that finished, by outcome. A later rollback does not count again.",
68 +});
69 +
70 +export const runDuration = Metric.timer("t3_run_duration", {
71 + description: "Wall time of a run, from when it started to its terminal status.",
72 +});
73 +
74 +export const runQueueWait = Metric.timer("t3_run_queue_wait", {
75 + description: "Time a run waited between being requested and starting.",
76 +});
77 +
78 export const gitCommandsTotal = Metric.counter("t3_git_commands_total", {
79 description: "Total git commands executed by the server runtime.",
80 });
apps/server/src/orchestration-v2/EventSink.ts
+45 -20
@@ -26,6 +26,7 @@ import * as SqlClient from "effect/sql/SqlClient";
26 import { replayAndBufferProjectedLiveEvents } from "./LiveStreamBudget.ts";
27 import type { UnsequencedProjectEvent } from "../persistence/OrchestrationEventStore.ts";
28 import { projectDomainEventForWire } from "./WireProjection.ts";
29 +import { makeRunOutcomeRecorder, type RunOutcome } from "./RunMetrics.ts";
30
31 import * as CommandReceiptStore from "./CommandReceiptStore.ts";
32 import * as EffectOutbox from "./EffectOutbox.ts";
@@ -243,8 +244,23 @@ const layerBase: Layer.Layer<
244 );
245 }
246 });
246 - const publishStoredEvents = (events: ReadonlyArray<OrchestrationV2StoredEvent>) =>
247 - eventStore.publishCommitted(events).pipe(Effect.andThen(publishLiveEvents(events)));
247 + const runOutcomes = makeRunOutcomeRecorder((threadId) =>
248 + // One payload row; unlike the shell, it still resolves a deleted child thread.
249 + projectionStore
250 + .getThread(threadId)
251 + .pipe(Effect.map((thread) => thread.lineage.relationshipToParent === "subagent")),
252 + );
253 + // Recorded before publication, so an interrupted publish cannot lose a committed outcome.
254 + const publishStoredEvents = (
255 + events: ReadonlyArray<OrchestrationV2StoredEvent>,
256 + outcomes: ReadonlyArray<RunOutcome>,
257 + ) =>
258 + runOutcomes
259 + .record(outcomes)
260 + .pipe(
261 + Effect.andThen(eventStore.publishCommitted(events)),
262 + Effect.andThen(publishLiveEvents(events)),
263 + );
264
265 // Transactions commit one at a time, but each writer publishes after its
266 // commit. If a writer is descheduled in between, a later commit reaches
@@ -373,6 +389,8 @@ const layerBase: Layer.Layer<
389 updated_at = excluded.updated_at
390 `;
391 }
392 + // Read lineage here, on the connection this transaction holds; see makeRunOutcomeRecorder.
393 + return yield* runOutcomes.prepare(storedEvents);
394 });
395
396 const writeEffect = Effect.fn("orchestrationV2.EventSink.write")(function* (
@@ -384,7 +402,7 @@ const layerBase: Layer.Layer<
402 "orchestration_v2.thread_id": input.events[0]?.threadId ?? null,
403 });
404
387 - return yield* commitThenPublish(
405 + const { committed } = yield* commitThenPublish(
406 Effect.gen(function* () {
407 // Check ownership in the same transaction that claims the native session.
408 const imported = input.unlessNativeThreadOwned;
@@ -398,7 +416,7 @@ const layerBase: Layer.Layer<
416 AND json_extract(payload_json, '$.nativeThreadRef.nativeId') = ${imported.nativeThreadRef.nativeId}
417 LIMIT 1
418 `;
401 - if (owners.length > 0) return [];
419 + if (owners.length > 0) return { committed: [], outcomes: [] };
420 }
421 // Checkpoint capture can finish after a cohort writer reads its parent run.
422 // Merge in this transaction so the event log and projection keep that completion.
@@ -433,18 +451,19 @@ const layerBase: Layer.Layer<
451 ...(input.commandId === undefined ? {} : { commandId: input.commandId }),
452 events: normalized,
453 });
436 - yield* applyStoredEvents(committed);
454 + const outcomes = yield* applyStoredEvents(committed);
455 yield* effectOutbox.enqueue(input.effects);
438 - return committed;
456 + return { committed, outcomes };
457 }).pipe(Effect.withTracerEnabled(false)),
440 - (storedEvents) =>
458 + ({ committed, outcomes }) =>
459 Effect.gen(function* () {
460 if (input.effects.length > 0) {
461 yield* effectOutbox.notifyAvailable(input.effects.length);
462 }
445 - yield* publishStoredEvents(storedEvents);
463 + yield* publishStoredEvents(committed, outcomes);
464 }),
465 );
466 + return committed;
467 });
468
469 const writeIfRunCurrentEffect = Effect.fn("orchestrationV2.EventSink.writeIfRunCurrent")(
@@ -491,9 +510,9 @@ const layerBase: Layer.Layer<
510 ...(input.commandId === undefined ? {} : { commandId: input.commandId }),
511 events: normalized,
512 });
494 - yield* applyStoredEvents(storedEvents);
513 + const outcomes = yield* applyStoredEvents(storedEvents);
514 yield* effectOutbox.enqueue(input.effects ?? []);
496 - return { committed: true as const, storedEvents };
515 + return { committed: true as const, storedEvents, outcomes };
516 }),
517 (result) =>
518 Effect.gen(function* () {
@@ -501,7 +520,7 @@ const layerBase: Layer.Layer<
520 if (input.effects !== undefined && input.effects.length > 0) {
521 yield* effectOutbox.notifyAvailable(input.effects.length);
522 }
504 - yield* publishStoredEvents(result.storedEvents);
523 + yield* publishStoredEvents(result.storedEvents, result.outcomes);
524 }),
525 );
526 },
@@ -556,10 +575,13 @@ const layerBase: Layer.Layer<
575 ...(input.commandId === undefined ? {} : { commandId: input.commandId }),
576 events: normalized,
577 });
559 - yield* applyStoredEvents(storedEvents);
560 - return { committed: true as const, storedEvents };
578 + const outcomes = yield* applyStoredEvents(storedEvents);
579 + return { committed: true as const, storedEvents, outcomes };
580 }),
562 - (result) => (result.committed ? publishStoredEvents(result.storedEvents) : Effect.void),
581 + (result) =>
582 + result.committed
583 + ? publishStoredEvents(result.storedEvents, result.outcomes)
584 + : Effect.void,
585 );
586 });
587
@@ -589,10 +611,13 @@ const layerBase: Layer.Layer<
611
612 const normalized = yield* normalizeEvents(input.events);
613 const storedEvents = yield* eventStore.append({ events: normalized });
592 - yield* applyStoredEvents(storedEvents);
593 - return { committed: true as const, storedEvents };
614 + const outcomes = yield* applyStoredEvents(storedEvents);
615 + return { committed: true as const, storedEvents, outcomes };
616 }),
595 - (result) => (result.committed ? publishStoredEvents(result.storedEvents) : Effect.void),
617 + (result) =>
618 + result.committed
619 + ? publishStoredEvents(result.storedEvents, result.outcomes)
620 + : Effect.void,
621 );
622 });
623
@@ -642,7 +667,7 @@ const layerBase: Layer.Layer<
667 new Error(`Command ${input.commandId} produced no orchestration events.`),
668 );
669 }
645 - yield* applyStoredEvents(storedEvents);
670 + const outcomes = yield* applyStoredEvents(storedEvents);
671 yield* effectOutbox.enqueue(input.effects);
672 const receipt: CommandReceiptStore.CommandReceiptV2 = {
673 commandId: input.commandId,
@@ -661,7 +686,7 @@ const layerBase: Layer.Layer<
686 threadId: input.threadId,
687 ...input.cancelUnsettledEffects,
688 });
664 - return { receipt, storedEvents, committed: true as const, cancelledEffectIds };
689 + return { receipt, storedEvents, outcomes, committed: true as const, cancelledEffectIds };
690 }),
691 (result) =>
692 Effect.gen(function* () {
@@ -669,7 +694,7 @@ const layerBase: Layer.Layer<
694 if (result.committed && input.effects.length > 0) {
695 yield* effectOutbox.notifyAvailable(input.effects.length);
696 }
672 - if (result.committed) yield* publishStoredEvents(result.storedEvents);
697 + if (result.committed) yield* publishStoredEvents(result.storedEvents, result.outcomes);
698 }),
699 );
700 return {
apps/server/src/orchestration-v2/RunMetrics.test.ts new
+284
@@ -0,0 +1,284 @@
1 +import { assert, describe, it } from "@effect/vitest";
2 +import {
3 + EventId,
4 + MessageId,
5 + ProjectId,
6 + ProviderInstanceId,
7 + RunId,
8 + ThreadId,
9 + type OrchestrationV2AppThread,
10 + type OrchestrationV2DomainEvent,
11 + type OrchestrationV2Run,
12 + type OrchestrationV2StoredEvent,
13 +} from "@t3tools/contracts";
14 +import * as DateTime from "effect/DateTime";
15 +import * as Deferred from "effect/Deferred";
16 +import * as Effect from "effect/Effect";
17 +import * as Fiber from "effect/Fiber";
18 +import * as Layer from "effect/Layer";
19 +import * as Metric from "effect/Metric";
20 +import * as Option from "effect/Option";
21 +import * as SqlClient from "effect/sql/SqlClient";
22 +
23 +import * as SqlitePersistence from "../persistence/Sqlite.ts";
24 +import * as EventSink from "./EventSink.ts";
25 +import * as EventStore from "./EventStore.ts";
26 +import * as ProjectionStore from "./ProjectionStore.ts";
27 +import { makeRunOutcomeRecorder } from "./RunMetrics.ts";
28 +
29 +const at = (iso: string) => DateTime.makeUnsafe(iso);
30 +const threadId = ThreadId.make("thread:metrics");
31 +
32 +// Each test labels its runs with its own provider, since the metric registry is shared.
33 +const runUpdated = (
34 + sequence: number,
35 + provider: string,
36 + run: Pick<OrchestrationV2Run, "id" | "status" | "startedAt" | "completedAt"> &
37 + Partial<OrchestrationV2Run>,
38 +): OrchestrationV2StoredEvent => ({
39 + sequence,
40 + commandId: null,
41 + event: {
42 + type: "run.updated",
43 + id: EventId.make(`event:${provider}:${sequence}`),
44 + threadId,
45 + occurredAt: at("2026-10-19T12:00:00.000Z"),
46 + payload: {
47 + threadId,
48 + ordinal: 1,
49 + providerInstanceId: ProviderInstanceId.make(provider),
50 + modelSelection: { instanceId: ProviderInstanceId.make(provider), model: "claude-opus-5-5" },
51 + requestedAt: at("2026-10-19T12:00:00.000Z"),
52 + ...run,
53 + } as OrchestrationV2Run,
54 + },
55 +});
56 +
57 +// Prepare inside the transaction, record after its commit, as the EventSink does.
58 +const commitWith =
59 + (recorder: ReturnType<typeof makeRunOutcomeRecorder<unknown>>) =>
60 + (events: ReadonlyArray<OrchestrationV2StoredEvent>) =>
61 + recorder.prepare(events).pipe(Effect.flatMap(recorder.record));
62 +
63 +const runSnapshots = (provider: string) =>
64 + Metric.snapshot.pipe(
65 + Effect.map((snapshots) =>
66 + snapshots.filter(
67 + (snapshot) =>
68 + snapshot.id.startsWith("t3_run") && snapshot.attributes?.["provider"] === provider,
69 + ),
70 + ),
71 + );
72 +
73 +describe("makeRunOutcomeRecorder", () => {
74 + it.effect("records a finished run once, labelled, with its queue wait and duration", () =>
75 + Effect.gen(function* () {
76 + // A subagent thread's first run carries no delegatedTaskId; only its lineage says delegated.
77 + const recorder = makeRunOutcomeRecorder(() => Effect.succeed(true));
78 + const record = commitWith(recorder);
79 + const finished = {
80 + id: RunId.make("run:metrics:once"),
81 + status: "completed" as const,
82 + startedAt: at("2026-10-19T12:00:30.000Z"),
83 + completedAt: at("2026-10-19T12:10:30.000Z"),
84 + };
85 +
86 + yield* record([runUpdated(1, "p1", { ...finished, status: "running", completedAt: null })]);
87 + // A transaction that rolls back prepares and never records; it must not reserve the run.
88 + yield* recorder.prepare([runUpdated(2, "p1", finished)]);
89 + yield* record([runUpdated(2, "p1", finished)]);
90 + // Later updates to a finished run, such as its checkpoint, are not another run.
91 + yield* record([runUpdated(3, "p1", finished)]);
92 +
93 + const snapshots = yield* runSnapshots("p1");
94 + const byId = (id: string) => snapshots.find((snapshot) => snapshot.id === id);
95 + const counter = byId("t3_runs_total");
96 + assert.strictEqual(counter?.type === "Counter" ? counter.state.count : undefined, 1);
97 + assert.deepStrictEqual(counter?.attributes, {
98 + provider: "p1",
99 + modelFamily: "claude",
100 + outcome: "completed",
101 + delegated: "true",
102 + });
103 + const histogramSum = (id: string) => {
104 + const snapshot = byId(id);
105 + return snapshot?.type === "Histogram" ? snapshot.state.sum : undefined;
106 + };
107 + assert.strictEqual(histogramSum("t3_run_queue_wait"), 30_000);
108 + assert.strictEqual(histogramSum("t3_run_duration"), 600_000);
109 + }),
110 + );
111 +
112 + it.effect("does not count a rollback of a run that already finished", () =>
113 + Effect.gen(function* () {
114 + // A fresh recorder stands in for a restart, which forgets the runs already counted.
115 + const record = commitWith(makeRunOutcomeRecorder(() => Effect.succeed(false)));
116 + yield* record([
117 + runUpdated(1, "p2", {
118 + id: RunId.make("run:metrics:rolled-back"),
119 + status: "rolled_back",
120 + startedAt: at("2026-10-19T12:00:30.000Z"),
121 + completedAt: at("2026-10-20T09:00:00.000Z"),
122 + }),
123 + ]);
124 +
125 + assert.deepStrictEqual(yield* runSnapshots("p2"), []);
126 + }),
127 + );
128 +});
129 +
130 +describe("EventSink run outcomes", () => {
131 + const parentId = ThreadId.make("thread:metrics:sink:parent");
132 + const childId = ThreadId.make("thread:metrics:sink:child");
133 + const provider = ProviderInstanceId.make("p-sink");
134 + const now = at("2026-10-19T12:00:00.000Z");
135 + const thread = (id: ThreadId, lineage: OrchestrationV2AppThread["lineage"]) =>
136 + ({
137 + type: "thread.created",
138 + id: EventId.make(`event:${id}:created`),
139 + threadId: id,
140 + occurredAt: now,
141 + payload: {
142 + createdBy: "user",
143 + creationSource: "web",
144 + id,
145 + projectId: ProjectId.make("project:metrics"),
146 + title: id,
147 + providerInstanceId: provider,
148 + modelSelection: { instanceId: provider, model: "gpt-6.1" },
149 + runtimeMode: "full-access",
150 + interactionMode: "default",
151 + branch: null,
152 + worktreePath: null,
153 + activeProviderThreadId: null,
154 + lineage,
155 + forkedFrom: null,
156 + createdAt: now,
157 + updatedAt: now,
158 + archivedAt: null,
159 + settledOverride: null,
160 + settledAt: null,
161 + lastVisitedAt: null,
162 + deletedAt: null,
163 + },
164 + }) satisfies OrchestrationV2DomainEvent;
165 + const runId = RunId.make("run:metrics:sink:child");
166 + const run = (status: OrchestrationV2Run["status"]): OrchestrationV2Run => ({
167 + id: runId,
168 + threadId: childId,
169 + ordinal: 1,
170 + providerInstanceId: provider,
171 + modelSelection: { instanceId: provider, model: "gpt-6.1" },
172 + providerThreadId: null,
173 + userMessageId: MessageId.make("message:metrics:sink"),
174 + rootNodeId: null,
175 + activeAttemptId: null,
176 + status,
177 + queuePosition: null,
178 + requestedAt: now,
179 + startedAt: at("2026-10-19T12:00:10.000Z"),
180 + completedAt: status === "running" ? null : at("2026-10-19T12:01:10.000Z"),
181 + checkpointId: null,
182 + contextHandoffId: null,
183 + });
184 + const runEvent = (type: "run.created" | "run.updated", status: OrchestrationV2Run["status"]) =>
185 + ({
186 + type,
187 + id: EventId.make(`event:${runId}:${status}`),
188 + threadId: childId,
189 + runId,
190 + occurredAt: now,
191 + payload: run(status),
192 + }) satisfies OrchestrationV2DomainEvent;
193 +
194 + it.effect("reads lineage inside the transaction, so a concurrent writer cannot deadlock it", () =>
195 + Effect.gen(function* () {
196 + const lookupEntered = yield* Deferred.make<boolean>();
197 + const releaseLookup = yield* Deferred.make<void>();
198 + const writerStarted = yield* Deferred.make<void>();
199 + const pausedLookup = Layer.effect(
200 + ProjectionStore.ProjectionStoreV2,
201 + Effect.gen(function* () {
202 + const store = yield* ProjectionStore.ProjectionStoreV2;
203 + const sql = yield* SqlClient.SqlClient;
204 + let paused = false;
205 + return {
206 + ...store,
207 + getThread: (id) =>
208 + Effect.gen(function* () {
209 + if (id === childId && !paused) {
210 + paused = true;
211 + const inTransaction = Option.isSome(
212 + yield* Effect.serviceOption(sql.transactionService),
213 + );
214 + yield* Deferred.succeed(lookupEntered, inTransaction);
215 + // Outside the transaction this read would wait on the writer it is racing.
216 + if (inTransaction) yield* Deferred.await(releaseLookup);
217 + }
218 + return yield* store.getThread(id);
219 + }),
220 + };
221 + }),
222 + );
223 + const layer = EventSink.layer.pipe(
224 + Layer.provideMerge(pausedLookup),
225 + Layer.provideMerge(
226 + Layer.mergeAll(EventStore.layer, ProjectionStore.layer).pipe(
227 + Layer.provideMerge(SqlitePersistence.layerMemory),
228 + ),
229 + ),
230 + );
231 +
232 + yield* Effect.gen(function* () {
233 + const sink = yield* EventSink.EventSinkV2;
234 + yield* sink.write({
235 + events: [
236 + thread(parentId, {
237 + parentThreadId: null,
238 + relationshipToParent: null,
239 + rootThreadId: parentId,
240 + }),
241 + thread(childId, {
242 + parentThreadId: parentId,
243 + relationshipToParent: "subagent",
244 + rootThreadId: parentId,
245 + }),
246 + runEvent("run.created", "running"),
247 + ],
248 + });
249 + const terminal = yield* sink
250 + .write({ events: [runEvent("run.updated", "completed")] })
251 + .pipe(Effect.forkScoped);
252 + assert.isTrue(yield* Deferred.await(lookupEntered), "lineage read outside the transaction");
253 + const unrelated = yield* Deferred.succeed(writerStarted, undefined).pipe(
254 + Effect.andThen(
255 + sink.write({
256 + events: [
257 + thread(ThreadId.make("thread:metrics:sink:other"), {
258 + parentThreadId: null,
259 + relationshipToParent: null,
260 + rootThreadId: ThreadId.make("thread:metrics:sink:other"),
261 + }),
262 + ],
263 + }),
264 + ),
265 + Effect.forkScoped,
266 + );
267 + yield* Deferred.await(writerStarted);
268 + yield* Deferred.succeed(releaseLookup, undefined);
269 +
270 + const [terminalEvents, unrelatedEvents] = [
271 + yield* Fiber.join(terminal),
272 + yield* Fiber.join(unrelated),
273 + ];
274 + assert.strictEqual(terminalEvents.length, 1);
275 + assert.strictEqual(unrelatedEvents.length, 1);
276 + const counter = (yield* runSnapshots("p-sink")).find(
277 + (snapshot) => snapshot.id === "t3_runs_total",
278 + );
279 + assert.strictEqual(counter?.type === "Counter" ? counter.state.count : undefined, 1);
280 + assert.strictEqual(counter?.attributes?.["delegated"], "true");
281 + }).pipe(Effect.provide(layer));
282 + }),
283 + );
284 +});
apps/server/src/orchestration-v2/RunMetrics.ts new
+113
@@ -0,0 +1,113 @@
1 +import type {
2 + OrchestrationV2Run,
3 + OrchestrationV2StoredEvent,
4 + RunId,
5 + ThreadId,
6 +} from "@t3tools/contracts";
7 +import * as DateTime from "effect/DateTime";
8 +import type * as Duration from "effect/Duration";
9 +import * as Effect from "effect/Effect";
10 +import * as Metric from "effect/Metric";
11 +
12 +import { normalizeModelMetricLabel } from "../observability/Attributes.ts";
13 +import {
14 + metricAttributes,
15 + runDuration,
16 + runQueueWait,
17 + runsTotal,
18 +} from "../observability/Metrics.ts";
19 +
20 +const REMEMBERED_RUNS = 4096;
21 +// A rollback rewrites runs that already finished, stamping the rollback time as their completion,
22 +// so it is not counted as an outcome. Declared here because ThreadManagementService imports
23 +// EventSink.
24 +const OUTCOMES: ReadonlySet<OrchestrationV2Run["status"]> = new Set([
25 + "completed",
26 + "failed",
27 + "cancelled",
28 + "interrupted",
29 +]);
30 +
31 +export interface RunOutcome {
32 + readonly runId: RunId;
33 + readonly attributes: ReturnType<typeof metricAttributes>;
34 + readonly queueWait: Duration.Duration | undefined;
35 + readonly duration: Duration.Duration | undefined;
36 +}
37 +
38 +/**
39 + * Records each run once, when a committed event first shows it finished. A finished run's row is
40 + * updated again later (its checkpoint, a settlement), so recently counted runs are remembered.
41 + *
42 + * A run carries `delegatedTaskId` only when a follow-up is associated with a task, so a subagent
43 + * thread's first run is recognized by its thread's lineage instead. A failed lookup only costs the
44 + * label: the events still commit and publish.
45 + *
46 + * `prepare` reads lineage, so the EventSink runs it inside the write transaction, on the
47 + * connection that transaction already holds. `record` runs after the commit, under the publish
48 + * lane, and must stay free of I/O: a read there waits for the writer connection while holding the
49 + * lane, and the next writer holds that connection while it waits for the lane.
50 + */
51 +export const makeRunOutcomeRecorder = <E>(
52 + isSubagentThread: (threadId: ThreadId) => Effect.Effect<boolean, E>,
53 +) => {
54 + const counted = new Set<string>();
55 + // Untraced: both run for every committed batch inside the EventSink operation's span.
56 + // Never reads or writes `counted`, so a transaction that rolls back reserves nothing.
57 + const prepare = Effect.fnUntraced(function* (events: ReadonlyArray<OrchestrationV2StoredEvent>) {
58 + const outcomes: Array<RunOutcome> = [];
59 + // Every finished run is prepared, in order, even one already counted: an earlier outcome in
60 + // this batch may evict it before `record` reaches it, so only `record` decides what counts.
61 + // Only the lineage read is shared.
62 + const delegatedThreads = new Map<ThreadId, boolean>();
63 + for (const { event } of events) {
64 + if (event.type !== "run.updated") continue;
65 + const run = event.payload;
66 + if (!OUTCOMES.has(run.status)) continue;
67 + let delegated = run.delegatedTaskId !== undefined || delegatedThreads.get(run.threadId);
68 + if (delegated === undefined) {
69 + delegated = yield* isSubagentThread(run.threadId).pipe(Effect.orElseSucceed(() => false));
70 + delegatedThreads.set(run.threadId, delegated);
71 + }
72 + outcomes.push({
73 + runId: run.id,
74 + attributes: metricAttributes({
75 + provider: run.providerInstanceId,
76 + modelFamily: normalizeModelMetricLabel(run.modelSelection.model),
77 + outcome: run.status,
78 + delegated,
79 + }),
80 + queueWait:
81 + run.startedAt === null ? undefined : DateTime.distance(run.requestedAt, run.startedAt),
82 + duration:
83 + run.startedAt === null || run.completedAt === null
84 + ? undefined
85 + : DateTime.distance(run.startedAt, run.completedAt),
86 + });
87 + }
88 + return outcomes;
89 + });
90 + // Uninterruptible so a run is never marked counted with only part of its measurements.
91 + const record = Effect.fnUntraced(function* (outcomes: ReadonlyArray<RunOutcome>) {
92 + for (const outcome of outcomes) {
93 + // Concurrent writers can prepare the same run; the first to publish counts it.
94 + if (counted.has(outcome.runId)) continue;
95 + counted.add(outcome.runId);
96 + if (counted.size > REMEMBERED_RUNS) counted.delete(counted.values().next().value!);
97 + yield* Metric.update(Metric.withAttributes(runsTotal, outcome.attributes), 1);
98 + if (outcome.queueWait !== undefined) {
99 + yield* Metric.update(
100 + Metric.withAttributes(runQueueWait, outcome.attributes),
101 + outcome.queueWait,
102 + );
103 + }
104 + if (outcome.duration !== undefined) {
105 + yield* Metric.update(
106 + Metric.withAttributes(runDuration, outcome.attributes),
107 + outcome.duration,
108 + );
109 + }
110 + }
111 + }, Effect.uninterruptible);
112 + return { prepare, record };
113 +};