@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

main
ts 892 lines 35.9 KB
Raw
1 import {
2 NodeId,
3 CommandId,
4 OrchestrationV2DomainEvent,
5 OrchestrationV2StoredEvent,
6 type OrchestrationV2PlanArtifact,
7 type OrchestrationV2Run,
8 type OrchestrationV2Subagent,
9 type OrchestrationV2ThreadProjection,
10 type ModelSelection,
11 ProviderInstanceId,
12 ProviderSessionId,
13 ProviderThreadId,
14 ProviderTurnId,
15 RawEventId,
16 RunAttemptId,
17 RunId,
18 type RuntimeRequestId,
19 ThreadId,
20 } from "@t3tools/contracts";
21 import * as Context from "effect/Context";
22 import * as DateTime from "effect/DateTime";
23 import * as Effect from "effect/Effect";
24 import * as Layer from "effect/Layer";
25 import * as Schema from "effect/Schema";
26 import * as SqlError from "effect/sql/SqlError";
27
28 import { AgentScope } from "@t3tools/shared/AgentScope";
29 import { modelSelectionsEqual } from "@t3tools/shared/model";
30 import {
31 ProviderTurnAnalytics,
32 type ProviderTurnAnalyticsContext,
33 } from "../telemetry/ProviderUsageAnalytics.ts";
34 import * as GitVcsDriver from "../vcs/GitVcsDriver.ts";
35 import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry.ts";
36 import { UsageLimitResetFallback } from "../usage/UsageLimitSources.ts";
37 import * as EventSink from "./EventSink.ts";
38 import * as ProjectionStore from "./ProjectionStore.ts";
39 import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
40 import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
41 import { makeProviderFailureTurnItem } from "@t3tools/provider-core/server/failure";
42 import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
43 import { stripUnservedToolOutputImageBytes } from "./toolOutputImageBytes.ts";
44 import { boundFileChangeForStorage } from "./WireProjection.ts";
45
46 export class ProviderEventNormalizeError extends Schema.TaggedError<ProviderEventNormalizeError>()(
47 "ProviderEventNormalizeError",
48 {
49 providerSessionId: ProviderSessionId,
50 threadId: ThreadId,
51 // The rejected payload cannot be validated again while reporting its normalization failure.
52 eventType: Schema.String,
53 cause: Schema.optional(Schema.Defect()),
54 },
55 ) {
56 override get message(): string {
57 return `Failed to normalize provider event ${this.eventType} for thread ${this.threadId}.`;
58 }
59 }
60
61 export class ProviderEventPublishError extends Schema.TaggedError<ProviderEventPublishError>()(
62 "ProviderEventPublishError",
63 {
64 providerSessionId: ProviderSessionId,
65 eventCount: Schema.Number,
66 cause: Schema.optional(Schema.Defect()),
67 },
68 ) {
69 override get message(): string {
70 return `Failed to publish ${this.eventCount} normalized provider event(s).`;
71 }
72 }
73
74 // A write that outlasts busy_timeout at BEGIN IMMEDIATE fails before anything
75 // commits, and the sink publishes only after a commit, so repeating the whole
76 // write is safe. Failing it would fail the run over a brief lock.
77 const isLockTimeout = (error: EventSink.EventSinkV2Error) =>
78 SqlError.isSqlError(error.cause) && error.cause.reason._tag === "LockTimeoutError";
79 const retryLockTimeout = <A, R>(write: Effect.Effect<A, EventSink.EventSinkV2Error, R>) =>
80 Effect.retry(write, { while: isLockTimeout, times: 2 });
81
82 export const ProviderEventIngestorV2Error = Schema.Union([
83 ProviderEventNormalizeError,
84 ProviderEventPublishError,
85 ]);
86 export type ProviderEventIngestorV2Error = typeof ProviderEventIngestorV2Error.Type;
87
88 type TodoListPlan = Extract<OrchestrationV2PlanArtifact, { readonly kind: "todo_list" }>;
89
90 function withPlanStepDurations(
91 plan: TodoListPlan,
92 previous: TodoListPlan | undefined,
93 occurredAt: DateTime.Utc,
94 ): TodoListPlan {
95 const occurredAtIso = DateTime.formatIso(occurredAt);
96 const occurredAtMs = DateTime.toEpochMillis(occurredAt);
97 const previousById = new Map(previous?.steps.map((step) => [step.id, step]));
98 // Provider step IDs may be positional. Changed text must not inherit another task's timing.
99 const previousStep = (step: TodoListPlan["steps"][number]) => {
100 const prior = previousById.get(step.id);
101 return prior?.text === step.text ? prior : undefined;
102 };
103 const hasNewCompletion = plan.steps.some((step) => {
104 const prior = previousStep(step);
105 return step.status === "completed" && prior?.status !== "completed";
106 });
107 let fallbackCompletionConsumed = false;
108
109 return {
110 ...plan,
111 steps: plan.steps.map((step) => {
112 const prior = previousStep(step);
113 const baseStep = { id: step.id, text: step.text, status: step.status };
114 if (step.status === "completed") {
115 if (prior?.status === "completed") {
116 return {
117 ...baseStep,
118 ...(prior.durationMs === undefined ? {} : { durationMs: prior.durationMs }),
119 };
120 }
121 const durationAnchorAt =
122 prior?.status === "running" || !fallbackCompletionConsumed
123 ? prior?.durationAnchorAt
124 : occurredAtIso;
125 fallbackCompletionConsumed = true;
126 const anchorMs =
127 durationAnchorAt === undefined ? occurredAtMs : Date.parse(durationAnchorAt);
128 const durationMs = Number.isFinite(anchorMs) ? Math.max(0, occurredAtMs - anchorMs) : 0;
129 return {
130 ...baseStep,
131 ...(durationMs > 0 ? { durationMs } : {}),
132 };
133 }
134 if (step.status === "running") {
135 return {
136 ...baseStep,
137 durationAnchorAt:
138 prior?.status === "running" ? (prior.durationAnchorAt ?? occurredAtIso) : occurredAtIso,
139 };
140 }
141 return {
142 ...baseStep,
143 durationAnchorAt:
144 hasNewCompletion || prior?.status !== "pending"
145 ? occurredAtIso
146 : (prior.durationAnchorAt ?? occurredAtIso),
147 };
148 }),
149 };
150 }
151
152 const CONVERSATION_RUN_STATUSES = new Set<OrchestrationV2Run["status"]>([
153 "completed",
154 "interrupted",
155 "failed",
156 ]);
157
158 function runsOffNativeBranch(input: {
159 readonly projection: Pick<OrchestrationV2ThreadProjection, "runs" | "attempts" | "providerTurns">;
160 readonly providerThreadId: ProviderThreadId;
161 readonly retainedNativeTurnIds: ReadonlySet<string>;
162 readonly reportingRunId: RunId | undefined;
163 }): ReadonlyArray<OrchestrationV2Run> {
164 const { attempts, providerTurns, runs } = input.projection;
165 const nativeTurnId = (run: OrchestrationV2Run) => {
166 const attempt = attempts.find((candidate) => candidate.id === run.activeAttemptId);
167 if (attempt === undefined) return null;
168 const ref = providerTurns.find(
169 (turn) => turn.id === attempt.providerTurnId || turn.runAttemptId === attempt.id,
170 )?.nativeTurnRef;
171 return ref?.strength === "strong" ? ref.nativeId : null;
172 };
173 const settled = runs.flatMap((run) =>
174 run.providerThreadId === input.providerThreadId &&
175 run.id !== input.reportingRunId &&
176 CONVERSATION_RUN_STATUSES.has(run.status)
177 ? [{ run, nativeTurnId: nativeTurnId(run) }]
178 : [],
179 );
180 const lastRetainedOrdinal = settled.reduce(
181 (last, { run, nativeTurnId }) =>
182 nativeTurnId !== null && input.retainedNativeTurnIds.has(nativeTurnId)
183 ? Math.max(last, run.ordinal)
184 : last,
185 0,
186 );
187 return settled
188 .filter(({ run, nativeTurnId }) =>
189 nativeTurnId === null
190 ? run.ordinal > lastRetainedOrdinal
191 : !input.retainedNativeTurnIds.has(nativeTurnId),
192 )
193 .map(({ run }) => run);
194 }
195
196 export interface ProviderEventIngestInput {
197 readonly providerSessionId: ProviderSessionId;
198 readonly providerInstanceId: ProviderInstanceId;
199 readonly commandId?: CommandId;
200 readonly threadId: ThreadId;
201 readonly runId?: RunId;
202 readonly nodeId?: NodeId;
203 readonly rawEventId?: RawEventId;
204 readonly event: ProviderAdapter.ProviderAdapterV2Event;
205 readonly analyticsContext?: ProviderTurnAnalyticsContext;
206 }
207
208 export interface ProviderEventIngestorV2Shape {
209 readonly normalize: (
210 input: ProviderEventIngestInput,
211 ) => Effect.Effect<ReadonlyArray<OrchestrationV2DomainEvent>, ProviderEventIngestorV2Error>;
212 readonly ingestNormalized: (
213 input: ProviderEventIngestInput & {
214 readonly writeIfRunCurrent?: {
215 readonly runId: RunId;
216 readonly activeAttemptId: RunAttemptId;
217 readonly expectedStatus: OrchestrationV2Run["status"];
218 };
219 readonly writeIfProviderThreadOwner?: {
220 readonly providerThreadId: ProviderThreadId;
221 readonly runId: RunId;
222 readonly activeAttemptId: RunAttemptId;
223 readonly expectedLastRunOrdinal: number;
224 };
225 },
226 ) => Effect.Effect<ReadonlyArray<OrchestrationV2StoredEvent>, ProviderEventIngestorV2Error>;
227 }
228
229 export class ProviderEventIngestorV2 extends Context.Service<
230 ProviderEventIngestorV2,
231 ProviderEventIngestorV2Shape
232 >()("t3/orchestration-v2/ProviderEventIngestor/ProviderEventIngestorV2") {}
233
234 function compactUndefined<T extends Record<string, unknown>>(record: T): T {
235 return Object.fromEntries(Object.entries(record).filter(([, value]) => value !== undefined)) as T;
236 }
237
238 const decodeDomainEvent = Schema.decodeUnknownEffect(OrchestrationV2DomainEvent);
239
240 export const layer: Layer.Layer<
241 ProviderEventIngestorV2,
242 never,
243 | EventSink.EventSinkV2
244 | GitVcsDriver.GitVcsDriver
245 | IdAllocator.IdAllocatorV2
246 | ProjectionStore.ProjectionStoreV2
247 | ThreadCommandExecutor.ThreadCommandExecutor
248 | ProviderInstanceRegistry.ProviderInstanceRegistry
249 > = Layer.effect(
250 ProviderEventIngestorV2,
251 Effect.gen(function* () {
252 const eventSink = yield* EventSink.EventSinkV2;
253 const git = yield* GitVcsDriver.GitVcsDriver;
254 const projections = yield* ProjectionStore.ProjectionStoreV2;
255 const idAllocator = yield* IdAllocator.IdAllocatorV2;
256 const threadCommands = yield* ThreadCommandExecutor.ThreadCommandExecutor;
257 const analytics = yield* ProviderTurnAnalytics;
258 const agentScope = yield* AgentScope;
259 const instances = yield* ProviderInstanceRegistry.ProviderInstanceRegistry;
260 const usageLimitResets = yield* UsageLimitResetFallback;
261 const scope = yield* Effect.scope;
262 const completedTurnAnalytics = new Set<string>();
263
264 const makeDomainEvent =
265 /* This helper runs for individual orchestration events; the enclosing operation owns the span. */
266 Effect.fnUntraced(function* (
267 input: ProviderEventIngestInput,
268 payloadInput: {
269 readonly type: OrchestrationV2DomainEvent["type"];
270 readonly payload: OrchestrationV2DomainEvent["payload"];
271 readonly threadId?: ThreadId;
272 readonly runId?: RunId | null;
273 readonly nodeId?: NodeId | null;
274 readonly occurredAt?: DateTime.Utc;
275 },
276 ) {
277 const threadId = payloadInput.threadId ?? input.threadId;
278 const eventId = yield* idAllocator.allocate.event({
279 threadId,
280 providerSessionId: input.providerSessionId,
281 });
282 const occurredAt = payloadInput.occurredAt ?? (yield* DateTime.now);
283 return yield* decodeDomainEvent(
284 compactUndefined({
285 id: eventId,
286 type: payloadInput.type,
287 threadId,
288 runId: payloadInput.runId ?? input.runId,
289 nodeId: payloadInput.nodeId ?? input.nodeId,
290 driver: input.event.driver,
291 providerInstanceId: input.providerInstanceId,
292 rawEventId: input.rawEventId,
293 occurredAt,
294 payload: payloadInput.payload,
295 }),
296 );
297 });
298
299 const dismissNativeUserInputs = Effect.fn("ProviderEventIngestor.dismissNativeUserInputs")(
300 function* (
301 input: ProviderEventIngestInput,
302 providerTurnId: ProviderTurnId,
303 threadId = input.threadId,
304 retained: ReadonlyArray<RuntimeRequestId> = [],
305 ) {
306 const pending = yield* projections.getPendingNativeUserInputs(threadId, providerTurnId);
307 const now = yield* DateTime.now;
308 const events: Array<OrchestrationV2DomainEvent> = [];
309 const kept = new Set(retained);
310 for (const request of pending.runtimeRequests) {
311 if (kept.has(request.id)) continue;
312 events.push(
313 yield* makeDomainEvent(input, {
314 type: "runtime-request.updated",
315 threadId,
316 nodeId: request.nodeId,
317 payload: { ...request, status: "cancelled", resolvedAt: now },
318 }),
319 );
320 }
321 for (const node of pending.nodes) {
322 if (node.runtimeRequestId !== null && kept.has(node.runtimeRequestId)) continue;
323 events.push(
324 yield* makeDomainEvent(input, {
325 type: "node.updated",
326 threadId,
327 nodeId: node.id,
328 runId: node.runId,
329 payload: { ...node, status: "cancelled", completedAt: now },
330 }),
331 );
332 }
333 for (const item of pending.turnItems) {
334 if (item.type === "user_input_request" && kept.has(item.requestId)) continue;
335 events.push(
336 yield* makeDomainEvent(input, {
337 type: "turn-item.updated",
338 threadId,
339 nodeId: item.nodeId,
340 runId: item.runId,
341 payload: { ...item, status: "cancelled", completedAt: now, updatedAt: now },
342 }),
343 );
344 }
345 return events;
346 },
347 );
348
349 /** Providers may reveal native subagent models late; lock full-row updates against concurrent thread commands. */
350 const syncSubagentThreadModel = Effect.fn("ProviderEventIngestor.syncSubagentThreadModel")(
351 function* (input: ProviderEventIngestInput, subagent: OrchestrationV2Subagent) {
352 const { childThreadId, model } = subagent;
353 if (
354 subagent.origin !== "provider_native" ||
355 childThreadId === null ||
356 (model === null && subagent.modelSelection === undefined)
357 ) {
358 return [];
359 }
360 const reportedSelection = ({ modelSelection }: { modelSelection: ModelSelection }) =>
361 subagent.modelSelection ??
362 (model === null || model === modelSelection.model
363 ? modelSelection
364 : { instanceId: modelSelection.instanceId, model });
365 const staleThread = projections.getThread(childThreadId).pipe(
366 Effect.map((thread) =>
367 modelSelectionsEqual(thread.modelSelection, reportedSelection(thread)) ? null : thread,
368 ),
369 Effect.catchTags({ ProjectionStoreThreadNotFoundError: () => Effect.succeed(null) }),
370 );
371 if ((yield* staleThread) === null) return [];
372 return yield* threadCommands.withLock(
373 childThreadId,
374 Effect.gen(function* () {
375 const thread = yield* staleThread;
376 if (thread === null) return [];
377 const now = yield* DateTime.now;
378 const event = yield* makeDomainEvent(input, {
379 type: "thread.model-selection-updated",
380 threadId: thread.id,
381 payload: {
382 ...thread,
383 modelSelection: reportedSelection(thread),
384 updatedAt: now,
385 },
386 occurredAt: now,
387 });
388 return yield* eventSink.write({ events: [event] }).pipe(retryLockTimeout);
389 }),
390 );
391 },
392 );
393
394 /** Retire rewound runs like CheckpointRollbackService; retain the reporting run and its lastRunOrdinal. */
395 const rollBackRewoundRuns = Effect.fn("ProviderEventIngestor.rollBackRewoundRuns")(function* (
396 input: ProviderEventIngestInput,
397 rewind: {
398 readonly threadId: ThreadId;
399 readonly providerThreadId: ProviderThreadId;
400 readonly retainedNativeTurnIds: ReadonlySet<string>;
401 },
402 ) {
403 const projection = yield* projections.getThreadRecords(rewind.threadId, [
404 "runs",
405 "attempts",
406 "providerTurns",
407 "nodes",
408 "checkpoints",
409 ]);
410 const runs = runsOffNativeBranch({
411 projection,
412 providerThreadId: rewind.providerThreadId,
413 retainedNativeTurnIds: rewind.retainedNativeTurnIds,
414 reportingRunId: input.runId,
415 });
416 if (runs.length === 0) return [];
417 const now = yield* DateTime.now;
418 const eventId = () =>
419 idAllocator.allocate.event({
420 threadId: rewind.threadId,
421 providerSessionId: input.providerSessionId,
422 });
423 const runIds = new Set(runs.map((run) => run.id));
424 const events: Array<OrchestrationV2DomainEvent> = [];
425 for (const checkpoint of projection.checkpoints) {
426 if (checkpoint.status !== "ready" || checkpoint.runId === null) continue;
427 if (!runIds.has(checkpoint.runId)) continue;
428 events.push({
429 id: yield* eventId(),
430 type: "checkpoint.captured",
431 threadId: rewind.threadId,
432 runId: checkpoint.runId,
433 nodeId: checkpoint.nodeId,
434 providerInstanceId: input.providerInstanceId,
435 occurredAt: now,
436 payload: { ...checkpoint, status: "stale" },
437 });
438 }
439 for (const run of runs) {
440 const rootNode = projection.nodes.find((node) => node.id === run.rootNodeId);
441 events.push({
442 id: yield* eventId(),
443 type: "run.updated",
444 threadId: rewind.threadId,
445 runId: run.id,
446 ...(rootNode === undefined ? {} : { nodeId: rootNode.id }),
447 providerInstanceId: run.providerInstanceId,
448 occurredAt: now,
449 payload: { ...run, status: "rolled_back", completedAt: now },
450 });
451 if (rootNode !== undefined) {
452 events.push({
453 id: yield* eventId(),
454 type: "node.updated",
455 threadId: rewind.threadId,
456 runId: run.id,
457 nodeId: rootNode.id,
458 providerInstanceId: run.providerInstanceId,
459 occurredAt: now,
460 payload: { ...rootNode, status: "rolled_back", completedAt: now },
461 });
462 }
463 }
464 return events;
465 });
466
467 const normalize: ProviderEventIngestorV2Shape["normalize"] =
468 /* This helper runs for individual orchestration events; the enclosing operation owns the span. */
469 Effect.fnUntraced(
470 function* (input) {
471 switch (input.event.type) {
472 case "app_thread.created": {
473 const offered = input.event.appThread;
474 const existing = yield* projections.findThread(offered.id);
475 if (
476 existing !== null &&
477 (input.event.createOnly === true ||
478 (existing.title === offered.title &&
479 modelSelectionsEqual(existing.modelSelection, offered.modelSelection)))
480 )
481 return [];
482 const now = yield* DateTime.now;
483 return [
484 yield* makeDomainEvent(input, {
485 type: existing === null ? "thread.created" : "thread.metadata-updated",
486 threadId: offered.id,
487 payload:
488 existing === null
489 ? offered
490 : {
491 ...existing,
492 title: offered.title,
493 modelSelection: offered.modelSelection,
494 updatedAt: now,
495 },
496 occurredAt: now,
497 }),
498 ];
499 }
500 case "provider_session.updated":
501 return [
502 yield* makeDomainEvent(input, {
503 type: "provider-session.updated",
504 payload: input.event.providerSession,
505 }),
506 ];
507 case "provider_thread.updated": {
508 const threadId = input.event.providerThread.appThreadId ?? input.threadId;
509 const updated = yield* makeDomainEvent(input, {
510 type: "provider-thread.updated",
511 threadId,
512 payload: input.event.providerThread,
513 });
514 return input.event.retainedNativeTurnIds === undefined
515 ? [updated]
516 : [
517 updated,
518 ...(yield* rollBackRewoundRuns(input, {
519 threadId,
520 providerThreadId: input.event.providerThread.id,
521 retainedNativeTurnIds: new Set(input.event.retainedNativeTurnIds),
522 })),
523 ];
524 }
525 case "provider_turn.updated":
526 return [
527 ...(["completed", "interrupted", "failed", "cancelled"].includes(
528 input.event.providerTurn.status,
529 )
530 ? yield* dismissNativeUserInputs(
531 input,
532 input.event.providerTurn.id,
533 input.event.threadId,
534 input.event.retainedRuntimeRequestIds,
535 )
536 : []),
537 yield* makeDomainEvent(input, {
538 type: "provider-turn.updated",
539 ...(input.event.threadId === undefined ? {} : { threadId: input.event.threadId }),
540 payload: input.event.providerTurn,
541 nodeId: input.event.providerTurn.nodeId,
542 }),
543 ];
544 case "node.updated":
545 // Adapters send an unchanged node again with each text delta. Compare with the
546 // stored node, not the last one sent, because the server also writes nodes.
547 if (yield* projections.isNodeStored(input.event.node)) return [];
548 return [
549 yield* makeDomainEvent(input, {
550 type: "node.updated",
551 threadId: input.event.node.threadId,
552 payload: input.event.node,
553 runId: input.event.node.runId,
554 nodeId: input.event.node.id,
555 }),
556 ];
557 case "subagent.updated":
558 return [
559 yield* makeDomainEvent(input, {
560 type: "subagent.updated",
561 threadId: input.event.subagent.threadId,
562 payload: input.event.subagent,
563 runId: input.event.subagent.runId,
564 nodeId: input.event.subagent.id,
565 }),
566 ];
567 case "message.updated":
568 return [
569 yield* makeDomainEvent(input, {
570 type: "message.updated",
571 threadId: input.event.message.threadId,
572 payload: input.event.message,
573 runId: input.event.message.runId,
574 nodeId: input.event.message.nodeId,
575 }),
576 ];
577 case "turn_item.updated":
578 return [
579 yield* makeDomainEvent(input, {
580 type: "turn-item.updated",
581 threadId: input.event.turnItem.threadId,
582 payload: boundFileChangeForStorage(
583 stripUnservedToolOutputImageBytes(input.event.turnItem),
584 ),
585 runId: input.event.turnItem.runId,
586 nodeId: input.event.turnItem.nodeId,
587 }),
588 ];
589 case "runtime_request.updated":
590 return [
591 yield* makeDomainEvent(input, {
592 type: "runtime-request.updated",
593 ...(input.event.threadId === undefined ? {} : { threadId: input.event.threadId }),
594 payload: input.event.runtimeRequest,
595 nodeId: input.event.runtimeRequest.nodeId,
596 }),
597 ];
598 case "plan.updated": {
599 const occurredAt = yield* DateTime.now;
600 const plan = input.event.plan;
601 const previous =
602 plan.kind === "todo_list"
603 ? yield* projections.getPlan(plan.threadId, plan.id)
604 : undefined;
605 const payload =
606 plan.kind === "todo_list"
607 ? withPlanStepDurations(
608 plan,
609 previous?.kind === "todo_list" ? previous : undefined,
610 occurredAt,
611 )
612 : plan;
613 return [
614 yield* makeDomainEvent(input, {
615 type: "plan.updated",
616 threadId: plan.threadId,
617 payload,
618 runId: plan.runId,
619 nodeId: plan.nodeId,
620 occurredAt,
621 }),
622 ];
623 }
624 case "steer.updated": {
625 const item = yield* projections.getTurnItem({
626 threadId: input.event.threadId,
627 itemId: idAllocator.derive.userTurnItem({ messageId: input.event.messageId }),
628 });
629 if (
630 item?.type !== "user_message" ||
631 item.status === input.event.status ||
632 (item.status !== "pending" && item.status !== "completed")
633 ) {
634 return [];
635 }
636 const occurredAt = yield* DateTime.now;
637 return [
638 yield* makeDomainEvent(input, {
639 type: "turn-item.updated",
640 threadId: item.threadId,
641 payload: {
642 ...item,
643 status: input.event.status,
644 completedAt: input.event.status === "pending" ? null : occurredAt,
645 updatedAt: occurredAt,
646 },
647 runId: item.runId,
648 nodeId: item.nodeId,
649 occurredAt,
650 }),
651 ];
652 }
653 case "background_work.updated":
654 return [];
655 case "command_output.delta":
656 case "prompt_suggestion.updated":
657 // The session manager serves these to clients before ingestion.
658 return [];
659 case "turn.terminal":
660 const dismissed = yield* dismissNativeUserInputs(
661 input,
662 input.event.providerTurnId,
663 undefined,
664 input.event.retainedRuntimeRequestIds,
665 );
666 if (input.event.status !== "failed") {
667 return dismissed;
668 }
669 const occurredAt = yield* DateTime.now;
670 const reported = (yield* agentScope.oomKilled(input.threadId))
671 ? { ...input.event.failure, message: "Killed: out of memory.", code: "oom_kill" }
672 : input.event.failure;
673 // A proxied CLI can drop the reset time; the hub still knows when its pool refills.
674 const endpoint =
675 reported.class === "usage_limit" && reported.resetAt == null
676 ? (yield* Effect.flatMap(
677 instances.getInstance(input.providerInstanceId),
678 (instance) => instance?.snapshot.getSnapshot ?? Effect.undefined,
679 ))?.auth.endpoint
680 : undefined;
681 const hubResetAt =
682 endpoint === undefined
683 ? null
684 : yield* Effect.gen(function* () {
685 // The failing run's model, which may differ from a selection changed mid-turn.
686 const { runs } =
687 input.runId === undefined
688 ? { runs: [] }
689 : yield* projections.getThreadRecords(input.threadId, ["runs"], {
690 runIds: [input.runId],
691 });
692 return (
693 runs[0]?.modelSelection ??
694 (yield* projections.getThread(input.threadId)).modelSelection
695 ).model;
696 }).pipe(
697 Effect.orElseSucceed(() => null),
698 Effect.flatMap((model) =>
699 usageLimitResets.resetAt(input.event.driver, endpoint, model),
700 ),
701 );
702 const failure = hubResetAt === null ? reported : { ...reported, resetAt: hubResetAt };
703 return [
704 ...dismissed,
705 yield* makeDomainEvent(input, {
706 type: "turn-item.updated",
707 payload: makeProviderFailureTurnItem({
708 driver: input.event.driver,
709 threadId: input.threadId,
710 runId: input.runId ?? null,
711 nodeId: input.nodeId ?? null,
712 providerThreadId: input.event.providerThreadId,
713 providerTurnId: input.event.providerTurnId,
714 itemOrdinal: input.event.failureItemOrdinal,
715 failure,
716 ...(input.event.retry === undefined ? {} : { retry: input.event.retry }),
717 ...(input.event.retryStartedAt === undefined
718 ? {}
719 : { retryStartedAt: input.event.retryStartedAt }),
720 occurredAt,
721 }),
722 }),
723 ];
724 }
725 },
726 (effect, input) =>
727 effect.pipe(
728 Effect.mapError(
729 (cause) =>
730 new ProviderEventNormalizeError({
731 providerSessionId: input.providerSessionId,
732 threadId: input.threadId,
733 eventType: input.event.type,
734 cause,
735 }),
736 ),
737 ),
738 );
739
740 return ProviderEventIngestorV2.of({
741 normalize,
742 ingestNormalized:
743 /* This helper runs for individual orchestration events; the enclosing operation owns the span. */
744 Effect.fnUntraced(
745 function* (input) {
746 const events = yield* normalize(input);
747 if (events.length === 0) {
748 return [];
749 }
750 const mapWriteError = (cause: unknown) =>
751 new ProviderEventPublishError({
752 providerSessionId: input.providerSessionId,
753 eventCount: events.length,
754 cause,
755 });
756 if (input.writeIfProviderThreadOwner !== undefined) {
757 const ownerResult = yield* eventSink
758 .writeIfProviderThreadOwner({
759 guardPendingUserInputCancellations: true,
760 ...(input.commandId === undefined ? {} : { commandId: input.commandId }),
761 ...input.writeIfProviderThreadOwner,
762 events,
763 })
764 .pipe(retryLockTimeout, Effect.mapError(mapWriteError));
765 return ownerResult.storedEvents;
766 }
767 if (input.writeIfRunCurrent === undefined) {
768 return yield* eventSink
769 .write({
770 guardPendingUserInputCancellations: true,
771 ...(input.commandId === undefined ? {} : { commandId: input.commandId }),
772 events,
773 })
774 .pipe(retryLockTimeout, Effect.mapError(mapWriteError));
775 }
776 const result = yield* eventSink
777 .writeIfRunCurrent({
778 guardPendingUserInputCancellations: true,
779 ...(input.commandId === undefined ? {} : { commandId: input.commandId }),
780 threadId: input.threadId,
781 ...input.writeIfRunCurrent,
782 events,
783 })
784 .pipe(retryLockTimeout, Effect.mapError(mapWriteError));
785 return result.storedEvents;
786 },
787 (effect, input) =>
788 effect.pipe(
789 Effect.tap((storedEvents) =>
790 Effect.gen(function* () {
791 if (
792 storedEvents.length === 0 ||
793 input.event.type !== "turn.terminal" ||
794 input.event.status !== "failed" ||
795 input.event.failure.class !== "usage_limit"
796 )
797 return;
798 const instance = yield* instances.getInstance(input.providerInstanceId);
799 if (!instance?.snapshot.refreshUsage) return;
800 yield* instance.snapshot.refreshUsage.pipe(
801 Effect.ignoreCause({ log: true }),
802 Effect.forkIn(scope, { startImmediately: true }),
803 );
804 }),
805 ),
806 Effect.flatMap((storedEvents) =>
807 Effect.gen(function* () {
808 if (input.event.type !== "app_thread.created") return storedEvents;
809 const created = storedEvents.find(
810 (stored) => stored.event.type === "thread.created",
811 );
812 if (created?.event.type !== "thread.created") return storedEvents;
813 const child = created.event.payload;
814 if (child.worktreePath === null || child.branch === null) return storedEvents;
815 // Git renames before scanning committed threads. If that scan missed this
816 // creation, HEAD already has the new name, even if the parent moved away.
817 const branch = yield* git.resolveBranchName(child.worktreePath, "HEAD");
818 if (branch === null || branch === child.branch) return storedEvents;
819 const now = yield* DateTime.now;
820 const event = yield* makeDomainEvent(input, {
821 type: "thread.metadata-updated",
822 threadId: child.id,
823 payload: { ...child, branch, updatedAt: now },
824 occurredAt: now,
825 });
826 const synced = yield* eventSink.write({ events: [event] });
827 return [...storedEvents, ...synced];
828 }).pipe(
829 Effect.catchCause((cause) =>
830 Effect.logWarning("Native thread worktree branch recheck failed", {
831 threadId:
832 input.event.type === "app_thread.created"
833 ? input.event.appThread.id
834 : input.threadId,
835 cause,
836 }).pipe(Effect.as(storedEvents)),
837 ),
838 ),
839 ),
840 // @effect-diagnostics-next-line flatMapConditionalToFilterOrFail:off - The predicate narrows the captured event for the subagent update.
841 Effect.flatMap((storedEvents) =>
842 storedEvents.length === 0 || input.event.type !== "subagent.updated"
843 ? Effect.succeed(storedEvents)
844 : syncSubagentThreadModel(input, input.event.subagent).pipe(
845 Effect.map((synced) => [...storedEvents, ...synced]),
846 Effect.mapError(
847 (cause) =>
848 new ProviderEventPublishError({
849 providerSessionId: input.providerSessionId,
850 eventCount: 1,
851 cause,
852 }),
853 ),
854 ),
855 ),
856 Effect.tap((storedEvents) =>
857 Effect.gen(function* () {
858 if (storedEvents.length === 0 || input.event.type !== "provider_turn.updated")
859 return;
860 const providerTurn = input.event.providerTurn;
861 if (
862 providerTurn.status !== "completed" &&
863 providerTurn.status !== "failed" &&
864 providerTurn.status !== "interrupted" &&
865 providerTurn.status !== "cancelled"
866 )
867 return;
868 const key = `${input.providerInstanceId}:${providerTurn.id}`;
869 if (completedTurnAnalytics.has(key)) return;
870 completedTurnAnalytics.add(key);
871 if (completedTurnAnalytics.size > 4096) {
872 const oldest = completedTurnAnalytics.values().next().value;
873 if (oldest !== undefined) completedTurnAnalytics.delete(oldest);
874 }
875 yield* analytics.turnCompleted({
876 driver: input.event.driver,
877 providerTurn,
878 ...(input.analyticsContext === undefined
879 ? {}
880 : { context: input.analyticsContext }),
881 });
882 }),
883 ),
884 (effect) =>
885 input.event.type === "app_thread.created"
886 ? threadCommands.withLock(input.event.appThread.id, effect)
887 : effect,
888 ),
889 ),
890 });
891 }),
892 );