@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 1,007 lines 36.8 KB
Raw
1 import type { AgentAwarenessThread } from "../orchestration-v2/ProjectionStore.ts";
2 import {
3 isListedThreadLineage,
4 type EnvironmentId,
5 type OrchestrationV2DomainEvent,
6 type OrchestrationV2TurnItem,
7 type Project,
8 type RuntimeRequestId,
9 type ThreadId,
10 } from "@t3tools/contracts";
11 import {
12 RelayApi,
13 type RelayAgentActivityPublishProofPayload,
14 type RelayAgentActivityState,
15 } from "@t3tools/contracts/relay";
16 import {
17 type SubagentRequestAwareness,
18 projectThreadAwarenessV2,
19 subagentAwaitsUser,
20 turnItemCanChangeUserWait,
21 } from "@t3tools/shared/agentAwareness";
22 import { turnItemUpdateCanEndBackgroundWork } from "@t3tools/shared/orchestrationV2PendingBackgroundWork";
23 import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker";
24 import { withRelayClientTracing } from "@t3tools/shared/relayTracing";
25 import { normalizeRelayIssuer, RELAY_ACTIVITY_PUBLISH_TYP } from "@t3tools/shared/relayJwt";
26 import * as Cause from "effect/Cause";
27 import * as Context from "effect/Context";
28 import * as SecureRandom from "effect/crypto/SecureRandom";
29 import * as DateTime from "effect/DateTime";
30 import * as Effect from "effect/Effect";
31 import * as Fiber from "effect/Fiber";
32 import * as Layer from "effect/Layer";
33 import * as Option from "effect/Option";
34 import * as Queue from "effect/Queue";
35 import * as Ref from "effect/Ref";
36 import type * as Scope from "effect/Scope";
37 import * as Stream from "effect/Stream";
38 import * as FetchHttpClient from "effect/http/FetchHttpClient";
39 import * as HttpClient from "effect/http/HttpClient";
40 import * as HttpClientRequest from "effect/http/HttpClientRequest";
41 import * as HttpApiClient from "effect/http-api/HttpApiClient";
42
43 import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
44 import {
45 isAgentActivityPublishingEnabledValue,
46 PUBLISH_AGENT_ACTIVITY_SECRET,
47 RELAY_ENVIRONMENT_CREDENTIAL_SECRET,
48 RELAY_ISSUER_SECRET,
49 RELAY_URL_SECRET,
50 } from "../cloud/config.ts";
51 import { EnvironmentSigningKey } from "../cloud/environmentKeys.ts";
52 import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
53 import * as ThreadManagement from "../orchestration-v2/ThreadManagementService.ts";
54 import { isSnoozed } from "../orchestration-v2/ThreadSettlementService.ts";
55 import * as ProjectService from "../project/ProjectService.ts";
56 import { forkParked } from "../serverActivation.ts";
57
58 export class AgentAwarenessRelay extends Context.Service<
59 AgentAwarenessRelay,
60 {
61 readonly publishThread: (threadId: ThreadId) => Effect.Effect<void>;
62 readonly drain: Effect.Effect<void>;
63 readonly requestCatchUp: () => Effect.Effect<void>;
64 readonly start: () => Effect.Effect<void, never, Scope.Scope>;
65 }
66 >()("t3/relay/AgentAwarenessRelay") {}
67
68 function eventThreadId(event: OrchestrationV2DomainEvent): ThreadId {
69 return event.threadId;
70 }
71
72 function isTurnItemPayload(
73 payload: unknown,
74 ): payload is Pick<OrchestrationV2TurnItem, "type" | "status"> {
75 return (
76 typeof payload === "object" && payload !== null && "type" in payload && "status" in payload
77 );
78 }
79
80 function shouldPublishAgentAwarenessEvent(
81 event: Pick<OrchestrationV2DomainEvent, "type"> & { readonly payload?: unknown },
82 ): boolean {
83 if (
84 event.type === "thread.created" &&
85 typeof event.payload === "object" &&
86 event.payload !== null &&
87 "historyOrigin" in event.payload &&
88 event.payload.historyOrigin === "v1_import"
89 ) {
90 return false;
91 }
92 switch (event.type) {
93 case "thread.created":
94 case "thread.archived":
95 case "thread.unarchived":
96 case "thread.deleted":
97 case "thread.metadata-updated":
98 case "thread.pull-request-link-updated":
99 case "thread.pull-request-synced":
100 case "thread.issue-synced":
101 case "thread.settled":
102 case "thread.model-selection-updated":
103 case "thread.provider-switched":
104 case "run.created":
105 case "run.updated":
106 case "runtime-request.updated":
107 case "subagent.updated":
108 case "provider-thread.updated":
109 return true;
110 // Unsettling must not re-alert a dismissed result.
111 case "thread.unsettled":
112 case "thread.guest-share-updated":
113 case "thread.snoozed":
114 case "thread.unsnoozed":
115 case "thread.auto-settle-set":
116 case "thread.pinned":
117 case "thread.unpinned":
118 case "thread.pin-reordered":
119 case "thread.active-reordered":
120 case "thread.tags-updated":
121 case "thread.visited":
122 case "thread.visit-recorded":
123 case "thread.marked-unread":
124 case "thread.runtime-mode-updated":
125 case "thread.interaction-mode-updated":
126 case "run.background-work-cancelled":
127 case "run.output-submitted":
128 case "run-attempt.created":
129 case "run-attempt.updated":
130 case "node.updated":
131 case "provider-session.attached":
132 case "provider-session.updated":
133 case "provider-session.detached":
134 case "provider-turn.updated":
135 case "turn-item.updated":
136 return (
137 isTurnItemPayload(event.payload) &&
138 (turnItemUpdateCanEndBackgroundWork(event.payload) ||
139 turnItemCanChangeUserWait(event.payload))
140 );
141 case "message.updated":
142 case "plan.updated":
143 case "checkpoint-scope.created":
144 case "checkpoint.captured":
145 case "checkpoint.rollback-requested":
146 case "context-handoff.updated":
147 case "context-transfer.created":
148 case "context-transfer.updated":
149 return false;
150 }
151 }
152
153 export const makeAgentAwarenessPublishWorker = Effect.fnUntraced(function* <R>(
154 publish: (threadId: ThreadId) => Effect.Effect<void, never, R>,
155 ) {
156 const queued = new Set<ThreadId>();
157 const worker = yield* makeDrainableWorker((threadId: ThreadId) =>
158 Effect.sync(() => queued.delete(threadId)).pipe(Effect.andThen(publish(threadId))),
159 );
160 const enqueue = (threadId: ThreadId) =>
161 Effect.suspend(() => {
162 if (queued.has(threadId)) return Effect.void;
163 // Remove queued IDs when processing starts so publication can queue one later update.
164 queued.add(threadId);
165 return worker.enqueue(threadId);
166 }).pipe(Effect.uninterruptible);
167 return { enqueue, drain: worker.drain };
168 });
169
170 function agentAwarenessPublishIdentity(
171 state: RelayAgentActivityState | null,
172 subagentRequest: SubagentRequestAwareness | null = null,
173 ): string {
174 if (state === null) {
175 return "null";
176 }
177 const { updatedAt: _updatedAt, ...meaningfulState } = state;
178 // A child's next request reads the same as its last one, but it is a new
179 // wait and must reach the phone.
180 return JSON.stringify(
181 subagentRequest === null ? meaningfulState : [meaningfulState, subagentRequest.requestId],
182 );
183 }
184
185 const CLEARED_PUBLISH_IDENTITY = agentAwarenessPublishIdentity(null);
186
187 // A published tombstone stays recorded so repeats are skipped, but it is not live state.
188 function hasLivePublishedState(
189 publishedStateByThread: ReadonlyMap<ThreadId, string>,
190 threadId: ThreadId,
191 ): boolean {
192 const identity = publishedStateByThread.get(threadId);
193 return identity !== undefined && identity !== CLEARED_PUBLISH_IDENTITY;
194 }
195
196 function resolveAgentActivityPublishingStartupState(input: {
197 readonly relayConfigured: boolean;
198 readonly publishEnabled: boolean;
199 }): "waiting-for-link" | "disabled" | "enabled" {
200 if (!input.relayConfigured) {
201 return "waiting-for-link";
202 }
203 return input.publishEnabled ? "enabled" : "disabled";
204 }
205
206 const RELAY_AGENT_ACTIVITY_DETAIL_MAX_LENGTH = 160;
207 const REDACTED_RELAY_AGENT_FAILURE_DETAIL = "The agent run failed.";
208 const RELAY_AGENT_ACTIVITY_RETRY_DELAYS_MS = [1_000, 2_000, 4_000, 8_000, 16_000] as const;
209 // Nothing else re-queues a thread when the network returns, so a lost terminal state is retried until it lands.
210 const RELAY_AGENT_ACTIVITY_STEADY_RETRY_DELAY_MS = 60_000;
211
212 function sanitizeRelayAgentActivityState(
213 state: RelayAgentActivityState | null,
214 ): RelayAgentActivityState | null {
215 if (state === null) {
216 return null;
217 }
218 const { detail: _detail, ...rest } = state;
219 const detail = (state.phase === "failed" ? REDACTED_RELAY_AGENT_FAILURE_DETAIL : state.detail)
220 ?.trim()
221 .slice(0, RELAY_AGENT_ACTIVITY_DETAIL_MAX_LENGTH)
222 .trim();
223 return detail ? { ...rest, detail } : rest;
224 }
225
226 function relayEnvironmentClient(token: string) {
227 return HttpClient.mapRequest(HttpClientRequest.setHeader("authorization", `Bearer ${token}`));
228 }
229
230 function deliveryStats(
231 deliveries: ReadonlyArray<{
232 readonly ok: boolean;
233 readonly queued?: boolean | undefined;
234 readonly kind: string;
235 readonly apnsStatus?: number | null;
236 readonly apnsReason?: string | null;
237 }>,
238 ) {
239 let queued = 0;
240 let successful = 0;
241 let failed = 0;
242 const failedReasons: string[] = [];
243 const kinds = new Set<string>();
244
245 for (const delivery of deliveries) {
246 kinds.add(delivery.kind);
247 if (delivery.queued) {
248 queued += 1;
249 continue;
250 }
251 if (delivery.ok) {
252 successful += 1;
253 continue;
254 }
255 failed += 1;
256 failedReasons.push(`${delivery.apnsStatus ?? "transport"}:${delivery.apnsReason ?? "unknown"}`);
257 }
258
259 return {
260 total: deliveries.length,
261 queued,
262 successful,
263 failed,
264 kinds: [...kinds],
265 failedReasons,
266 };
267 }
268
269 const makePublishProof = Effect.fn("makePublishProof")(function* (input: {
270 readonly relayIssuer: string;
271 readonly environmentId: string;
272 readonly threadId: ThreadId;
273 readonly state: RelayAgentActivityState | null;
274 readonly jti: string;
275 }) {
276 const now = yield* DateTime.now;
277 const expiresAt = DateTime.add(now, { minutes: 5 });
278 const payload = {
279 iss: `t3-env:${input.environmentId}`,
280 aud: normalizeRelayIssuer(input.relayIssuer),
281 sub: input.environmentId,
282 jti: input.jti,
283 iat: Math.floor(now.epochMilliseconds / 1_000),
284 exp: Math.floor(expiresAt.epochMilliseconds / 1_000),
285 environmentId: input.environmentId as RelayAgentActivityPublishProofPayload["environmentId"],
286 threadId: input.threadId,
287 state: input.state,
288 } satisfies RelayAgentActivityPublishProofPayload;
289 const signingKey = yield* EnvironmentSigningKey;
290 return yield* signingKey.sign({ typ: RELAY_ACTIVITY_PUBLISH_TYP, payload });
291 });
292
293 function describeThreadShellForAwareness(
294 thread: Option.Option<AgentAwarenessThread>,
295 ): Record<string, unknown> {
296 if (Option.isNone(thread)) {
297 return { found: false };
298 }
299 const shell = thread.value;
300 return {
301 found: true,
302 status: shell.status,
303 activityRunStatus: shell.activityRunStatus ?? null,
304 activeRunId: shell.activeRunId ?? null,
305 latestRunId: shell.latestRunId ?? null,
306 pendingRuntimeRequestKind: shell.pendingRuntimeRequest?.kind ?? null,
307 hasActionableProposedPlan: shell.hasActionableProposedPlan,
308 };
309 }
310
311 interface SubagentRequest {
312 readonly topLevelThreadId: ThreadId;
313 readonly requestId: RuntimeRequestId;
314 }
315
316 interface SubagentAlert {
317 readonly requestId: RuntimeRequestId;
318 readonly updatedAt: string;
319 }
320
321 const waitingApprovalId = (thread: AgentAwarenessThread | null) =>
322 thread !== null && thread.archivedAt === null && subagentAwaitsUser(thread)
323 ? thread.pendingRuntimeRequest?.id
324 : undefined;
325
326 function resolveAgentAwarenessRelayPublishSnapshot(input: {
327 readonly environmentId: EnvironmentId;
328 readonly thread: Option.Option<AgentAwarenessThread>;
329 readonly project: Option.Option<Project>;
330 readonly pendingSubagentRequest: SubagentRequestAwareness | null;
331 }): {
332 readonly projectId: string | null;
333 readonly state: RelayAgentActivityState | null;
334 /** The child request behind the state's wait, when the parent isn't waiting itself. */
335 readonly subagentWait: SubagentRequestAwareness | null;
336 readonly reason: "snapshot" | "thread-not-found" | "project-not-found";
337 } {
338 if (Option.isNone(input.thread)) {
339 return {
340 projectId: null,
341 state: null,
342 subagentWait: null,
343 reason: "thread-not-found",
344 };
345 }
346 if (Option.isNone(input.project)) {
347 return {
348 projectId: input.thread.value.projectId,
349 state: null,
350 subagentWait: null,
351 reason: "project-not-found",
352 };
353 }
354 const awarenessInput = {
355 environmentId: input.environmentId,
356 project: input.project.value,
357 thread: input.thread.value,
358 };
359 const state = projectThreadAwarenessV2({
360 ...awarenessInput,
361 pendingSubagentRequest: input.pendingSubagentRequest,
362 });
363 const childDrivesWait =
364 input.pendingSubagentRequest !== null &&
365 state?.phase !== projectThreadAwarenessV2(awarenessInput)?.phase;
366 return {
367 projectId: input.thread.value.projectId,
368 state: snoozedCompletion(input.thread.value, state)
369 ? null
370 : sanitizeRelayAgentActivityState(state),
371 subagentWait: childDrivesWait ? input.pendingSubagentRequest : null,
372 reason: "snapshot",
373 };
374 }
375
376 // Checked at completion time, so a completion processed after the wake stays silent too.
377 function snoozedCompletion(
378 thread: AgentAwarenessThread,
379 state: { readonly phase: string } | null,
380 ): boolean {
381 if (state?.phase !== "completed" || thread.latestRunCompletedAt == null) return false;
382 const completedAt = DateTime.toEpochMillis(thread.latestRunCompletedAt);
383 if (thread.snoozedAt != null && completedAt < DateTime.toEpochMillis(thread.snoozedAt)) {
384 return false;
385 }
386 return isSnoozed(thread, completedAt);
387 }
388
389 function terminalWorkSinceStart(thread: AgentAwarenessThread, startedAt: number): boolean {
390 return (
391 thread.latestRunCompletedAt != null &&
392 DateTime.toEpochMillis(thread.latestRunCompletedAt) > startedAt
393 );
394 }
395
396 export function resolveAgentAwarenessRelayActiveThreadIds(input: {
397 readonly environmentId: EnvironmentId;
398 readonly startedAt: number;
399 readonly projects: ReadonlyArray<Pick<Project, "id" | "title">>;
400 readonly threads: ReadonlyArray<AgentAwarenessThread>;
401 }): ReadonlyArray<ThreadId> {
402 const projectById = new Map(input.projects.map((project) => [project.id, project]));
403 return input.threads
404 .filter((thread) => {
405 // Publishing a waiting subagent republishes its top-level thread.
406 if (subagentAwaitsUser(thread)) return true;
407 const project = projectById.get(thread.projectId);
408 if (!project) {
409 return false;
410 }
411 const state = projectThreadAwarenessV2({
412 environmentId: input.environmentId,
413 project,
414 thread,
415 });
416 return (
417 state !== null &&
418 (state.phase !== "completed" && state.phase !== "failed"
419 ? true
420 : terminalWorkSinceStart(thread, input.startedAt))
421 );
422 })
423 .map((thread) => thread.id);
424 }
425
426 /** @public */
427 export const make = Effect.gen(function* () {
428 const secrets = yield* ServerSecretStore.ServerSecretStore;
429 const serverEnvironment = yield* ServerEnvironment.ServerEnvironment;
430 const threads = yield* ThreadManagement.ThreadManagementService;
431 const projects = yield* ProjectService.ProjectService;
432 const random = yield* SecureRandom.SecureRandom;
433 const signingKey = yield* EnvironmentSigningKey;
434 const scope = yield* Effect.scope;
435 const startedAt = (yield* DateTime.now).epochMilliseconds;
436 const catchUpRequests = yield* Queue.dropping<void>(1);
437 const publishedStateByThreadRef = yield* Ref.make(new Map<ThreadId, string>());
438 // Android keys alerts by phase and time, so a child's unchanged request keeps
439 // its first alert time through parent renames, relinks and the parent's own requests.
440 const childAlertByThread = new Map<ThreadId, SubagentAlert>();
441 const subagentRequests = new Map<ThreadId, SubagentRequest>();
442
443 const readSecretString = (name: string) =>
444 secrets
445 .get(name)
446 .pipe(
447 Effect.map((bytes) =>
448 Option.isSome(bytes) ? new TextDecoder().decode(bytes.value) : null,
449 ),
450 );
451
452 const readRelayConfig = Effect.gen(function* () {
453 const [url, issuer, environmentCredential] = yield* Effect.all([
454 readSecretString(RELAY_URL_SECRET),
455 readSecretString(RELAY_ISSUER_SECRET),
456 readSecretString(RELAY_ENVIRONMENT_CREDENTIAL_SECRET),
457 ]);
458 return url && environmentCredential
459 ? { url, issuer: issuer ?? url, environmentCredential }
460 : null;
461 });
462
463 const readPublishAgentActivityEnabled = readSecretString(PUBLISH_AGENT_ACTIVITY_SECRET).pipe(
464 Effect.map(isAgentActivityPublishingEnabledValue),
465 );
466
467 const makeRelayClient = (relayConfig: {
468 readonly url: string;
469 readonly environmentCredential: string;
470 }) =>
471 HttpApiClient.make(RelayApi, {
472 baseUrl: relayConfig.url,
473 transformClient: relayEnvironmentClient(relayConfig.environmentCredential),
474 }).pipe(Effect.provide(FetchHttpClient.layer));
475
476 let cachedRelayClient: Effect.Success<ReturnType<typeof makeRelayClient>> | undefined;
477 let publishedRelayConfig: NonNullable<Effect.Success<typeof readRelayConfig>> | undefined;
478
479 // Confirm through the same worker so tombstones cannot race live updates; recovery clears the deadline.
480 const publishConfirmDeadlines = new Map<ThreadId, number>();
481 // Threads whose confirmation holds a completion, so a thread that settles during the hold still alerts.
482 const heldCompletions = new Set<ThreadId>();
483 let schedulePublishConfirm: (threadId: ThreadId) => Effect.Effect<void> = () => Effect.void;
484 const publishRetries = new Map<
485 ThreadId,
486 { readonly attempts: number; timer: Fiber.Fiber<void> | undefined }
487 >();
488 const cancelPublishRetry = Effect.fnUntraced(function* (threadId: ThreadId) {
489 const retry = publishRetries.get(threadId);
490 publishRetries.delete(threadId);
491 if (retry?.timer !== undefined) yield* Fiber.interrupt(retry.timer);
492 });
493 const cancelPublishRetries = Effect.suspend(() =>
494 Effect.forEach([...publishRetries.keys()], cancelPublishRetry, { discard: true }),
495 );
496 let schedulePublishRetry: (threadId: ThreadId) => Effect.Effect<void> = () => Effect.void;
497 let enqueueTopLevelPublish: (threadId: ThreadId) => Effect.Effect<void> = () => Effect.void;
498 const resetPublishedConnection = Effect.gen(function* () {
499 publishedRelayConfig = undefined;
500 cachedRelayClient = undefined;
501 publishConfirmDeadlines.clear();
502 heldCompletions.clear();
503 yield* Ref.set(publishedStateByThreadRef, new Map());
504 });
505
506 const publishThreadUnsafe = Effect.fn("publishThreadUnsafe")(function* (threadId: ThreadId) {
507 const publishAgentActivity = yield* readPublishAgentActivityEnabled;
508 if (!publishAgentActivity) {
509 yield* cancelPublishRetries;
510 yield* resetPublishedConnection;
511 yield* Effect.logDebug("agent activity publish skipped; publication disabled", {
512 threadId,
513 });
514 return;
515 }
516 const relayConfig = yield* readRelayConfig;
517 if (!relayConfig) {
518 yield* cancelPublishRetries;
519 yield* resetPublishedConnection;
520 yield* Effect.logDebug("agent activity publish skipped; relay link credentials unavailable", {
521 threadId,
522 });
523 return;
524 }
525 if (
526 publishedRelayConfig?.url !== relayConfig.url ||
527 publishedRelayConfig.issuer !== relayConfig.issuer ||
528 publishedRelayConfig.environmentCredential !== relayConfig.environmentCredential
529 ) {
530 yield* resetPublishedConnection;
531 publishedRelayConfig = relayConfig;
532 }
533 const environmentId = yield* serverEnvironment.getEnvironmentId;
534
535 const publishState = Effect.fn("relay.agent_activity.publish")(function* (input: {
536 readonly projectId: string | null;
537 readonly state: RelayAgentActivityState | null;
538 readonly reason: string;
539 }) {
540 if (cachedRelayClient === undefined) {
541 cachedRelayClient = yield* makeRelayClient(relayConfig);
542 }
543 const relayClient = cachedRelayClient;
544 const proof = yield* makePublishProof({
545 relayIssuer: relayConfig.issuer,
546 environmentId,
547 threadId,
548 state: input.state,
549 jti: yield* random.uuidV4(),
550 }).pipe(Effect.provideService(EnvironmentSigningKey, signingKey));
551
552 yield* Effect.logInfo("publishing agent activity for thread", {
553 environmentId,
554 threadId,
555 projectId: input.projectId,
556 statePhase: input.state?.phase ?? null,
557 hasState: input.state !== null,
558 reason: input.reason,
559 });
560
561 const response = yield* relayClient.server
562 .publishAgentActivity({
563 params: {
564 environmentId,
565 threadId,
566 },
567 payload: {
568 state: input.state,
569 proof,
570 },
571 })
572 .pipe(Effect.timeout("10 seconds"));
573
574 yield* Effect.logInfo("agent activity publish completed", {
575 environmentId,
576 threadId,
577 ok: response.ok,
578 deliveries: deliveryStats(response.deliveries),
579 });
580 });
581
582 // Read only this thread; full shell reads would make each activity event scale with all threads.
583 const threadShell = yield* readAwarenessThread(threadId);
584 yield* trackSubagentRequest(threadId, threadShell);
585 if (
586 threadShell !== null &&
587 !isListedThreadLineage(threadShell.lineage) &&
588 !hasLivePublishedState(yield* Ref.get(publishedStateByThreadRef), threadId)
589 ) {
590 // Subagents and side chats have no relay rows to clear; per-event tombstones would needlessly redeliver user aggregates.
591 return;
592 }
593 const thread =
594 threadShell === null || threadShell.archivedAt !== null
595 ? Option.none<AgentAwarenessThread>()
596 : Option.some(threadShell);
597 const project = Option.isSome(thread)
598 ? yield* projects.getById(thread.value.projectId)
599 : Option.none<Project>();
600 const snapshot = resolveAgentAwarenessRelayPublishSnapshot({
601 environmentId,
602 thread,
603 project,
604 pendingSubagentRequest: yield* currentSubagentRequestFor(threadId),
605 });
606 const publishIdentity = agentAwarenessPublishIdentity(snapshot.state, snapshot.subagentWait);
607 const publishedStateByThread = yield* Ref.get(publishedStateByThreadRef);
608 if (
609 (snapshot.state?.phase === "completed" || snapshot.state?.phase === "failed") &&
610 !hasLivePublishedState(publishedStateByThread, threadId)
611 ) {
612 if (Option.isNone(thread) || !terminalWorkSinceStart(thread.value, startedAt)) return;
613 }
614 if (
615 publishedStateByThread.get(threadId) === publishIdentity &&
616 // A held completion settled after an earlier clear still has to alert.
617 !(snapshot.state === null && heldCompletions.has(threadId))
618 ) {
619 // Clear moot deadlines so later transient nulls cannot skip their deferral window.
620 publishConfirmDeadlines.delete(threadId);
621 heldCompletions.delete(threadId);
622 yield* Effect.logDebug("agent activity publish skipped; projected state unchanged", {
623 environmentId,
624 threadId,
625 reason: snapshot.reason,
626 });
627 return;
628 }
629
630 // Confirm tombstones over live state to avoid removing active cards.
631 // Confirm completion: a run can end moments before its continuation starts, and the relay
632 // alerts on every move into completed.
633 const requiresConfirmation = snapshot.state === null || snapshot.state?.phase === "completed";
634 if (requiresConfirmation) {
635 if (snapshot.state?.phase === "completed") heldCompletions.add(threadId);
636 const nowMs = (yield* DateTime.now).epochMilliseconds;
637 const deadline = publishConfirmDeadlines.get(threadId);
638 if (deadline === undefined) {
639 publishConfirmDeadlines.set(threadId, nowMs + 5_000);
640 yield* Effect.logInfo("agent activity publish deferred pending confirmation", {
641 environmentId,
642 threadId,
643 reason: snapshot.reason,
644 statePhase: snapshot.state?.phase ?? null,
645 shell: describeThreadShellForAwareness(thread),
646 });
647 yield* schedulePublishConfirm(threadId);
648 return;
649 }
650 if (nowMs < deadline) {
651 return;
652 }
653 yield* Effect.logInfo("agent activity deferred publish confirmed", {
654 environmentId,
655 threadId,
656 reason: snapshot.reason,
657 statePhase: snapshot.state?.phase ?? null,
658 shell: describeThreadShellForAwareness(thread),
659 });
660 // Settled during the hold, not resumed: alert the current completion, then clear the card
661 // on a new window. Only settlement may hide it; queued or silent runs also project to null.
662 const unsettled =
663 snapshot.state === null &&
664 heldCompletions.has(threadId) &&
665 Option.isSome(thread) &&
666 thread.value.settledOverride === "settled"
667 ? resolveAgentAwarenessRelayPublishSnapshot({
668 environmentId,
669 thread: Option.some({ ...thread.value, settledOverride: null }),
670 project,
671 pendingSubagentRequest: null,
672 })
673 : null;
674 if (unsettled?.state?.phase === "completed") {
675 const completion = unsettled.state;
676 yield* publishState({
677 projectId: unsettled.projectId,
678 state: completion,
679 reason: unsettled.reason,
680 }).pipe(withRelayClientTracing);
681 heldCompletions.delete(threadId);
682 publishConfirmDeadlines.set(threadId, nowMs + 5_000);
683 yield* Ref.update(publishedStateByThreadRef, (publishedStates) =>
684 new Map(publishedStates).set(threadId, agentAwarenessPublishIdentity(completion)),
685 );
686 yield* schedulePublishConfirm(threadId);
687 return;
688 }
689 } else {
690 publishConfirmDeadlines.delete(threadId);
691 heldCompletions.delete(threadId);
692 }
693
694 if (snapshot.reason === "thread-not-found") {
695 yield* Effect.logDebug("publishing agent activity tombstone; thread not found", {
696 environmentId,
697 threadId,
698 });
699 } else if (snapshot.reason === "project-not-found") {
700 yield* Effect.logDebug("publishing agent activity tombstone; project not found", {
701 environmentId,
702 threadId,
703 projectId: snapshot.projectId,
704 });
705 }
706
707 const childRequestId = snapshot.subagentWait?.requestId;
708 const childAlert = childAlertByThread.get(threadId);
709 const state =
710 snapshot.state !== null &&
711 childRequestId !== undefined &&
712 childAlert?.requestId === childRequestId
713 ? { ...snapshot.state, updatedAt: childAlert.updatedAt }
714 : snapshot.state;
715 yield* publishState({
716 projectId: snapshot.projectId,
717 state,
718 reason: snapshot.reason,
719 }).pipe(withRelayClientTracing);
720 if (state !== null && childRequestId !== undefined) {
721 childAlertByThread.set(threadId, { requestId: childRequestId, updatedAt: state.updatedAt });
722 }
723 publishConfirmDeadlines.delete(threadId);
724 heldCompletions.delete(threadId);
725 yield* Ref.update(publishedStateByThreadRef, (publishedStates) =>
726 new Map(publishedStates).set(threadId, publishIdentity),
727 );
728 });
729
730 const readAwarenessThread = (threadId: ThreadId) =>
731 threads
732 .getAgentAwarenessSnapshot(threadId)
733 .pipe(Effect.map((snapshot) => snapshot.threads[0] ?? null));
734
735 // The subagent helpers below are untraced: they run per event inside
736 // publishThreadUnsafe, whose span already covers them.
737 const findTopLevelThreadId = Effect.fnUntraced(function* (child: AgentAwarenessThread) {
738 const seen = new Set<ThreadId>([child.id]);
739 let current: AgentAwarenessThread | null = child;
740 while (
741 current?.lineage.relationshipToParent === "subagent" ||
742 current?.lineage.sideChat === true
743 ) {
744 const parentId: ThreadId | null = current.lineage.parentThreadId;
745 if (parentId === null || seen.has(parentId)) return null;
746 seen.add(parentId);
747 current = yield* readAwarenessThread(parentId);
748 }
749 return current?.id ?? null;
750 });
751
752 const trackSubagentRequest = Effect.fnUntraced(function* (
753 threadId: ThreadId,
754 threadShell: AgentAwarenessThread | null,
755 ) {
756 const requestId = waitingApprovalId(threadShell);
757 const previous = subagentRequests.get(threadId);
758 // An unchanged wait still republishes its parent: a relink catch-up reaches it only here.
759 if (threadShell === null || requestId === undefined) {
760 if (previous === undefined) return;
761 subagentRequests.delete(threadId);
762 yield* enqueueTopLevelPublish(previous.topLevelThreadId);
763 return;
764 }
765 const topLevelThreadId =
766 previous?.topLevelThreadId ?? (yield* findTopLevelThreadId(threadShell));
767 if (topLevelThreadId === null) return;
768 subagentRequests.set(threadId, { topLevelThreadId, requestId });
769 yield* enqueueTopLevelPublish(topLevelThreadId);
770 });
771
772 // The map can miss a child's update while publishing is off or the relay is
773 // unlinked, so each waiting child is re-read before its parent is published.
774 const currentSubagentRequestFor = Effect.fnUntraced(function* (topLevelThreadId: ThreadId) {
775 let pending: SubagentRequestAwareness | null = null;
776 for (const [childId, entry] of subagentRequests) {
777 if (entry.topLevelThreadId !== topLevelThreadId) continue;
778 const child = yield* readAwarenessThread(childId);
779 const requestId = waitingApprovalId(child);
780 if (child === null || requestId === undefined) {
781 subagentRequests.delete(childId);
782 continue;
783 }
784 if (requestId !== entry.requestId) subagentRequests.set(childId, { ...entry, requestId });
785 // The newest wait is the one to announce.
786 if (pending === null || DateTime.isGreaterThan(child.updatedAt, pending.updatedAt)) {
787 pending = {
788 requestId,
789 updatedAt: child.updatedAt,
790 waitsForInput: child.pendingRuntimeRequest?.kind === "user_input",
791 };
792 }
793 }
794 return pending;
795 });
796
797 const processThreadPublish = Effect.fnUntraced(function* (threadId: ThreadId) {
798 const retry = publishRetries.get(threadId);
799 if (retry?.timer !== undefined) {
800 const timer = retry.timer;
801 retry.timer = undefined;
802 yield* Fiber.interrupt(timer);
803 }
804 yield* publishThreadUnsafe(threadId).pipe(
805 Effect.tap(() => cancelPublishRetry(threadId)),
806 Effect.catchCause((cause) => {
807 if (Cause.hasInterruptsOnly(cause)) return Effect.void;
808 return Effect.logWarning("agent activity publish failed", {
809 threadId,
810 cause: Cause.pretty(cause),
811 }).pipe(Effect.andThen(schedulePublishRetry(threadId)));
812 }),
813 Effect.withSpan("AgentAwarenessRelay.publishThread"),
814 );
815 });
816
817 const worker = yield* makeAgentAwarenessPublishWorker(processThreadPublish);
818 const enqueueThreadPublish = (threadId: ThreadId) =>
819 cancelPublishRetry(threadId).pipe(Effect.andThen(worker.enqueue(threadId)));
820 enqueueTopLevelPublish = enqueueThreadPublish;
821 const publishThread: AgentAwarenessRelay["Service"]["publishThread"] = (threadId) =>
822 enqueueThreadPublish(threadId).pipe(Effect.andThen(worker.drain));
823
824 schedulePublishRetry = Effect.fnUntraced(function* (threadId: ThreadId) {
825 const attempts = publishRetries.get(threadId)?.attempts ?? 0;
826 const delayMs =
827 RELAY_AGENT_ACTIVITY_RETRY_DELAYS_MS[attempts] ?? RELAY_AGENT_ACTIVITY_STEADY_RETRY_DELAY_MS;
828 if (attempts === RELAY_AGENT_ACTIVITY_RETRY_DELAYS_MS.length) {
829 yield* Effect.logWarning(
830 "agent activity publish retry budget exhausted; retrying every minute",
831 {
832 threadId,
833 },
834 );
835 }
836 const retry = {
837 attempts: attempts + 1,
838 timer: undefined as Fiber.Fiber<void> | undefined,
839 };
840 publishRetries.set(threadId, retry);
841 const timer = yield* Effect.sleep(delayMs).pipe(
842 Effect.andThen(
843 Effect.suspend(() => {
844 if (publishRetries.get(threadId) !== retry) return Effect.void;
845 retry.timer = undefined;
846 return worker.enqueue(threadId);
847 }),
848 ),
849 Effect.forkIn(scope),
850 );
851 if (publishRetries.get(threadId) !== retry) {
852 yield* Fiber.interrupt(timer);
853 } else {
854 retry.timer = timer;
855 }
856 });
857
858 const publishActiveThreadsUnsafe = Effect.gen(function* () {
859 const relayUrl = yield* readSecretString(RELAY_URL_SECRET).pipe(
860 Effect.orElseSucceed(() => null),
861 );
862 if (!relayUrl) {
863 yield* Effect.logDebug("agent activity snapshot skipped; relay link credentials unavailable");
864 return "unlinked" as const;
865 }
866 const publishAgentActivity = yield* readPublishAgentActivityEnabled.pipe(
867 Effect.orElseSucceed(() => false),
868 );
869 if (!publishAgentActivity) {
870 yield* Effect.logDebug("agent activity snapshot skipped; publication disabled");
871 return "disabled" as const;
872 }
873 const relayConfig = yield* readRelayConfig.pipe(Effect.orElseSucceed(() => null));
874 if (!relayConfig) {
875 yield* Effect.logDebug("agent activity snapshot skipped; relay link credentials unavailable");
876 return "unlinked" as const;
877 }
878 const environmentId = yield* serverEnvironment.getEnvironmentId;
879 const projectSnapshot = yield* projects.snapshot;
880 const activitySnapshot = yield* threads.getAgentAwarenessSnapshot();
881 // Live children aren't listed, but a waiting one makes its top-level thread wait.
882 const liveChildren = yield* Effect.forEach(
883 (yield* threads.listActiveShellIndex()).filter(
884 (thread) => thread.lineage.relationshipToParent === "subagent" && thread.mayBeLive,
885 ),
886 (thread) => readAwarenessThread(thread.id),
887 );
888 const activeThreadIds = resolveAgentAwarenessRelayActiveThreadIds({
889 environmentId,
890 startedAt,
891 projects: projectSnapshot.projects,
892 threads: [...activitySnapshot.threads, ...liveChildren.filter((thread) => thread !== null)],
893 });
894 if (activeThreadIds.length === 0) {
895 yield* Effect.logDebug("agent activity snapshot has no publishable threads");
896 return "published" as const;
897 }
898 yield* Effect.logInfo("publishing active agent activity snapshot", {
899 count: activeThreadIds.length,
900 });
901 yield* Effect.forEach(activeThreadIds, enqueueThreadPublish, { discard: true });
902 yield* worker.drain;
903 return "published" as const;
904 });
905
906 // Many environments never link, so while unlinked the retry backs off from 5
907 // s to 60 s. A linked environment keeps the 5 s retry, because `t3 connect
908 // publish` can turn publishing on from another process. Keep this untraced:
909 // an unlinked environment can retry for the lifetime of the process.
910 const publishActiveThreadsWhenConfigured = Effect.fnUntraced(function* (
911 logEnabledWhenReady: boolean,
912 ) {
913 let unlinkedRetryDelayMs = 5_000;
914 while (true) {
915 const result = yield* publishActiveThreadsUnsafe.pipe(
916 Effect.orElseSucceed(() => "failed" as const),
917 );
918 if (result === "published") {
919 if (logEnabledWhenReady) {
920 const relayConfig = yield* readRelayConfig.pipe(Effect.orElseSucceed(() => null));
921 yield* Effect.logInfo("agent activity publishing enabled after link reconciliation", {
922 relayUrl: relayConfig?.url,
923 });
924 }
925 logEnabledWhenReady = false;
926 unlinkedRetryDelayMs = 5_000;
927 yield* Queue.take(catchUpRequests);
928 continue;
929 }
930 const retryDelayMs = result === "unlinked" ? unlinkedRetryDelayMs : 5_000;
931 yield* Effect.race(Effect.sleep(retryDelayMs), Queue.take(catchUpRequests));
932 if (result === "unlinked") {
933 unlinkedRetryDelayMs = Math.min(unlinkedRetryDelayMs * 2, 60_000);
934 }
935 }
936 });
937
938 schedulePublishConfirm = (threadId) =>
939 Effect.forkIn(
940 Effect.sleep("5 seconds").pipe(
941 Effect.andThen(worker.enqueue(threadId)),
942 Effect.catchCause((cause) =>
943 Effect.logWarning("deferred agent activity confirmation failed", {
944 threadId,
945 cause: Cause.pretty(cause),
946 }),
947 ),
948 ),
949 scope,
950 ).pipe(Effect.asVoid);
951
952 const start: AgentAwarenessRelay["Service"]["start"] = Effect.fn("AgentAwarenessRelay.start")(
953 function* () {
954 const [relayConfig, publishEnabled] = yield* Effect.all([
955 readRelayConfig.pipe(Effect.orElseSucceed(() => null)),
956 readPublishAgentActivityEnabled.pipe(Effect.orElseSucceed(() => false)),
957 ]);
958 const startupState = resolveAgentActivityPublishingStartupState({
959 relayConfigured: relayConfig !== null,
960 publishEnabled,
961 });
962 switch (startupState) {
963 case "waiting-for-link":
964 yield* Effect.logInfo(
965 "agent activity publishing standby; waiting for T3 Connect link reconciliation",
966 );
967 break;
968 case "disabled":
969 yield* Effect.logInfo("agent activity publishing disabled by T3 Connect configuration");
970 break;
971 case "enabled":
972 yield* Effect.logInfo("agent activity publishing enabled", {
973 relayUrl: relayConfig?.url,
974 });
975 break;
976 }
977 yield* forkParked(
978 Effect.sleep("1 second").pipe(
979 Effect.andThen(publishActiveThreadsWhenConfigured(startupState !== "enabled")),
980 ),
981 );
982 yield* forkParked(
983 Stream.runForEach(threads.streamDomainEvents, (event) => {
984 const threadId = eventThreadId(event);
985 if (!shouldPublishAgentAwarenessEvent(event)) {
986 return Effect.void;
987 }
988 return Effect.logDebug("agent activity publishing queued thread publish", {
989 eventType: event.type,
990 threadId,
991 }).pipe(Effect.andThen(enqueueThreadPublish(threadId)));
992 }),
993 );
994 },
995 );
996
997 return AgentAwarenessRelay.of({
998 publishThread,
999 drain: worker.drain,
1000 requestCatchUp: Effect.fn("AgentAwarenessRelay.requestCatchUp")(function () {
1001 return Queue.offer(catchUpRequests, undefined).pipe(Effect.asVoid);
1002 }),
1003 start,
1004 });
1005 });
1006
1007 export const layer = Layer.effect(AgentAwarenessRelay, make);