@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 795 lines 25.9 KB
Raw
1 // @effect-diagnostics nodeBuiltinImport:off
2 /** Native and canonical views share write state to avoid races on the same thread file. */
3 import * as NodeCrypto from "node:crypto";
4 import * as NodeFS from "node:fs";
5 import * as NodePath from "node:path";
6
7 import type { ThreadId } from "@t3tools/contracts";
8 import type * as ProviderEventLoggers from "@t3tools/provider-core/server/ProviderEventLoggers";
9 import { makeAsyncRotatingFileSink } from "@t3tools/shared/logging";
10 import { errorTag } from "@t3tools/shared/observability";
11 import * as Clock from "effect/Clock";
12 import * as DateTime from "effect/DateTime";
13 import * as Effect from "effect/Effect";
14 import * as FileSystem from "effect/FileSystem";
15 import * as Path from "effect/Path";
16 import * as Result from "effect/Result";
17 import * as Exit from "effect/Exit";
18 import * as Schema from "effect/Schema";
19 import * as Scope from "effect/Scope";
20 import * as SynchronizedRef from "effect/SynchronizedRef";
21
22 import {
23 ATTACHMENT_ID_THREAD_SEGMENT_MAX_CHARS,
24 toSafeThreadAttachmentSegment,
25 } from "../attachmentStore.ts";
26 import type * as ResourceAttribution from "../resourceTelemetry/ResourceAttribution.ts";
27
28 const MEBIBYTE = 1024 * 1024;
29 const DAY_MS = 24 * 60 * 60 * 1_000;
30 const DEFAULT_MAX_BYTES = 10 * MEBIBYTE;
31 const DEFAULT_MAX_FILES = 10;
32 const DEFAULT_BATCH_WINDOW_MS = 1_000;
33 const DEFAULT_MAX_TOTAL_BYTES = 512 * MEBIBYTE;
34 const DEFAULT_MAX_AGE_MS = 14 * DAY_MS;
35 const DEFAULT_RETENTION_CHECK_INTERVAL_MS = 5 * 60 * 1_000;
36 const DEFAULT_MAX_BUFFERED_BYTES = MEBIBYTE;
37 const DEFAULT_MAX_BUFFERED_RECORDS = 512;
38 const MAX_RECORD_CHARACTERS = 64 * 1024;
39 const MAX_RECORD_FIELDS = 1_024;
40 const MAX_RECORD_DEPTH = 16;
41 const GLOBAL_THREAD_SEGMENT = "_global";
42 const LOG_SCOPE = "provider-observability";
43 const encodeUnknownJsonString = Schema.encodeUnknownEffect(Schema.fromJsonString(Schema.Unknown));
44
45 const transientCanonicalEventTypes = new Set([
46 "content.delta",
47 "hook.progress",
48 "item.updated",
49 "task.progress",
50 "thread.realtime.audio.delta",
51 "tool.progress",
52 "turn.proposed.delta",
53 ]);
54 const transientNativeMethods = new Set([
55 "item/agentMessage/delta",
56 "item/commandExecution/outputDelta",
57 "item/fileChange/outputDelta",
58 "item/plan/delta",
59 "item/reasoning/summaryTextDelta",
60 "item/reasoning/textDelta",
61 "thread/realtime/outputAudio/delta",
62 "thread/realtime/transcript/delta",
63 "turn/diff/updated",
64 ]);
65 const transientAcpUpdates = new Set(["agent_message_chunk", "agent_thought_chunk"]);
66 const transientCursorUpdates = new Set([
67 "text-delta",
68 "thinking-delta",
69 "token-delta",
70 "shell-output-delta",
71 "partial-tool-call",
72 ]);
73
74 export type EventNdjsonStream = "native" | "canonical" | "orchestration";
75
76 export interface EventNdjsonLogStore {
77 readonly filePath: string;
78 readonly logger: (stream: EventNdjsonStream) => ProviderEventLoggers.EventNdjsonLogger;
79 readonly close: () => Effect.Effect<void>;
80 }
81
82 export interface EventNdjsonLogStoreOptions {
83 readonly maxBytes?: number;
84 readonly maxFiles?: number;
85 readonly batchWindowMs?: number;
86 readonly attribution?: ResourceAttribution.ResourceAttribution["Service"];
87 }
88
89 export class EventNdjsonLogDirectoryError extends Schema.TaggedError<EventNdjsonLogDirectoryError>()(
90 "EventNdjsonLogDirectoryError",
91 {
92 directory: Schema.String,
93 cause: Schema.Defect(),
94 },
95 ) {
96 override get message(): string {
97 return `Failed to create provider event log directory '${this.directory}'`;
98 }
99 }
100
101 export type EventNdjsonLogStoreError = EventNdjsonLogDirectoryError;
102
103 interface ResolvedOptions {
104 readonly maxBytes: number;
105 readonly maxFiles: number;
106 readonly batchWindowMs: number;
107 readonly maxTotalBytes: number;
108 readonly maxAgeMs: number;
109 readonly retentionCheckIntervalMs: number;
110 readonly maxBufferedBytes: number;
111 readonly maxBufferedRecords: number;
112 readonly attribution: ResourceAttribution.ResourceAttribution["Service"] | undefined;
113 }
114
115 export interface PendingRecord {
116 readonly stream: EventNdjsonStream;
117 readonly threadSegment: string;
118 readonly line: string;
119 readonly bytes: number;
120 }
121
122 type AsyncFileSink = Effect.Success<ReturnType<typeof makeAsyncRotatingFileSink>>;
123
124 interface StoreState {
125 readonly pending: Array<PendingRecord>;
126 readonly pendingBytes: number;
127 readonly sinks: ReadonlyMap<string, AsyncFileSink>;
128 readonly writtenSinceRetention: ReadonlySet<string>;
129 readonly flushScheduled: boolean;
130 readonly closed: boolean;
131 readonly lastRetentionAt: number;
132 }
133
134 interface AttributionSummary {
135 readonly stream: EventNdjsonStream;
136 readonly count: number;
137 readonly logicalWriteBytes: number;
138 }
139
140 interface FileOperationFailure {
141 readonly filePath: string;
142 readonly cause: unknown;
143 }
144
145 interface RetentionResult {
146 readonly failures: ReadonlyArray<FileOperationFailure>;
147 }
148
149 interface DrainResult {
150 readonly attributions: ReadonlyArray<AttributionSummary>;
151 readonly failures: ReadonlyArray<FileOperationFailure>;
152 }
153
154 function logWarning(message: string, context: Record<string, unknown>): Effect.Effect<void> {
155 return Effect.logWarning(message, context).pipe(Effect.annotateLogs({ scope: LOG_SCOPE }));
156 }
157
158 function resolveThreadSegment(raw: string | null | undefined): string {
159 if (typeof raw !== "string") return GLOBAL_THREAD_SEGMENT;
160 const normalized = toSafeThreadAttachmentSegment(raw);
161 if (normalized === null) return GLOBAL_THREAD_SEGMENT;
162 // Delegated thread ids share a prefix longer than the segment, so ids that may be cut get a hash.
163 if (raw.length <= ATTACHMENT_ID_THREAD_SEGMENT_MAX_CHARS) return normalized;
164 const hash = NodeCrypto.createHash("sha256").update(raw).digest("hex").slice(0, 16);
165 return `${normalized}-${hash}`;
166 }
167
168 function resolveStreamLabel(stream: EventNdjsonStream): string {
169 return stream === "native" ? "NTIVE" : stream === "orchestration" ? "ORCH" : "CANON";
170 }
171
172 function providerLogPrefix(filePath: string): string {
173 const basename = NodePath.basename(filePath);
174 const extension = NodePath.extname(basename);
175 return `${extension.length > 0 ? basename.slice(0, -extension.length) : basename}.`;
176 }
177
178 function providerLogPath(directory: string, prefix: string, threadSegment: string): string {
179 return NodePath.join(directory, `${prefix}${threadSegment}.log`);
180 }
181
182 export function shouldPersistProviderEvent(stream: EventNdjsonStream, event: unknown): boolean {
183 if (stream === "orchestration" || typeof event !== "object" || event === null) {
184 return true;
185 }
186 try {
187 const type = Reflect.get(event, "type");
188 if (typeof type === "string" && transientCanonicalEventTypes.has(type)) {
189 return false;
190 }
191 if (stream !== "native") return true;
192
193 const nested = Reflect.get(event, "event");
194 const envelope = typeof nested === "object" && nested !== null ? nested : event;
195 // Decoded frames preserve raw-frame information without duplicating every token delta.
196 if (Reflect.get(envelope, "stage") === "raw") return false;
197 const decodedPayload = Reflect.get(envelope, "payload");
198 const nativeEvent =
199 Reflect.get(envelope, "stage") === "decoded" &&
200 typeof decodedPayload === "object" &&
201 decodedPayload !== null
202 ? decodedPayload
203 : envelope;
204 const method = Reflect.get(nativeEvent, "method");
205 if (
206 typeof method === "string" &&
207 (transientNativeMethods.has(method) ||
208 method.startsWith("claude/stream_event/content_block_delta/"))
209 ) {
210 return false;
211 }
212
213 const nativeType = Reflect.get(nativeEvent, "type");
214 if (nativeType === "system" && Reflect.get(nativeEvent, "subtype") === "thinking_tokens") {
215 return false;
216 }
217 if (nativeType === "message.part.delta") return false;
218 if (nativeType === "stream_event") {
219 const streamEvent = Reflect.get(nativeEvent, "event");
220 if (
221 typeof streamEvent === "object" &&
222 streamEvent !== null &&
223 Reflect.get(streamEvent, "type") === "content_block_delta"
224 ) {
225 return false;
226 }
227 }
228
229 if (nativeType === "interaction.update") {
230 let update = Reflect.get(nativeEvent, "update");
231 if (typeof update !== "object" || update === null) return true;
232 // Subagent updates arrive wrapped; keep the wrapper when it carries a lifecycle record.
233 if (Reflect.get(update, "type") === "tool-call-delta") {
234 update = Reflect.get(update, "taskUpdate");
235 if (typeof update !== "object" || update === null) return true;
236 }
237 const updateType = Reflect.get(update, "type");
238 return typeof updateType !== "string" || !transientCursorUpdates.has(updateType);
239 }
240
241 const payload = Reflect.get(nativeEvent, "payload");
242 if (typeof payload !== "object" || payload === null) return true;
243
244 if (method === "session/update") {
245 const update = Reflect.get(payload, "update");
246 if (typeof update !== "object" || update === null) return true;
247 const updateType = Reflect.get(update, "sessionUpdate");
248 return typeof updateType !== "string" || !transientAcpUpdates.has(updateType);
249 }
250
251 if (nativeType === "message.part.updated") {
252 const properties = Reflect.get(payload, "properties");
253 if (typeof properties !== "object" || properties === null) return true;
254 const part = Reflect.get(properties, "part");
255 if (typeof part !== "object" || part === null) return true;
256 const partType = Reflect.get(part, "type");
257 if (partType === "text" || partType === "reasoning") return false;
258 if (partType === "tool") {
259 // Running snapshots repeat growing output.
260 const state = Reflect.get(part, "state");
261 if (typeof state === "object" && state !== null) {
262 return Reflect.get(state, "status") !== "running";
263 }
264 }
265 return true;
266 }
267
268 return true;
269 } catch {
270 return true;
271 }
272 }
273
274 const summaryFields = [
275 "provider",
276 "protocol",
277 "kind",
278 "providerSessionId",
279 "direction",
280 "stage",
281 "type",
282 "subtype",
283 "method",
284 "id",
285 "threadId",
286 "turnId",
287 "requestId",
288 "session_id",
289 "status",
290 "is_error",
291 "api_error_status",
292 "terminal_reason",
293 "stop_reason",
294 "operation",
295 "code",
296 "willRetry",
297 "message",
298 "event",
299 "payload",
300 "params",
301 "result",
302 "thread",
303 "turn",
304 "error",
305 "turns",
306 "items",
307 "content",
308 ] as const;
309
310 function summarizeProviderEvent(event: unknown): unknown {
311 let remainingFields = 128;
312 let remainingCharacters = 8 * 1024;
313 const summarize = (value: unknown, depth: number): unknown => {
314 if (typeof value === "string") {
315 if (value.length > Math.min(1_024, remainingCharacters)) {
316 return { omittedCharacters: value.length };
317 }
318 remainingCharacters -= value.length;
319 return value;
320 }
321 if (value === null || typeof value === "number" || typeof value === "boolean") return value;
322 if (typeof value !== "object") return undefined;
323 if (Array.isArray(value)) return { itemCount: value.length };
324 const summary: Record<string, unknown> = { truncated: true };
325 if (depth >= 6) return summary;
326 for (const key of summaryFields) {
327 if (remainingFields <= 0) break;
328 const nested = Reflect.get(value, key);
329 if (nested === undefined) continue;
330 remainingFields -= 1;
331 summary[key] = summarize(nested, depth + 1);
332 }
333 return summary;
334 };
335 try {
336 return summarize(event, 0);
337 } catch {
338 return { truncated: true };
339 }
340 }
341
342 export function boundProviderEventForLogging(event: unknown): unknown {
343 let remainingCharacters = MAX_RECORD_CHARACTERS;
344 let remainingFields = MAX_RECORD_FIELDS;
345 const ancestors = new WeakSet<object>();
346 const fits = (value: unknown, depth: number): boolean => {
347 if (typeof value === "string") {
348 remainingCharacters -= value.length;
349 return remainingCharacters >= 0;
350 }
351 if (typeof value !== "object" || value === null) return true;
352 if (depth > MAX_RECORD_DEPTH || ancestors.has(value)) return false;
353 if (Array.isArray(value) && value.length > remainingFields) return false;
354 ancestors.add(value);
355 for (const key in value) {
356 if (!Object.hasOwn(value, key)) continue;
357 remainingFields -= 1;
358 remainingCharacters -= key.length;
359 if (
360 remainingFields < 0 ||
361 remainingCharacters < 0 ||
362 !fits(Reflect.get(value, key), depth + 1)
363 )
364 return false;
365 }
366 ancestors.delete(value);
367 return true;
368 };
369 try {
370 if (fits(event, 0)) return event;
371 } catch {}
372 return summarizeProviderEvent(event);
373 }
374
375 export const writeBatchedMessages = Effect.fnUntraced(function* (
376 sink: AsyncFileSink,
377 records: ReadonlyArray<PendingRecord>,
378 maxBytes: number,
379 onWritten: (records: ReadonlyArray<PendingRecord>) => void,
380 ) {
381 let pendingRecords: Array<PendingRecord> = [];
382 let pendingBytes = 0;
383
384 const flush = Effect.gen(function* () {
385 if (pendingRecords.length === 0) return;
386 const writtenRecords = pendingRecords;
387 yield* sink.write(writtenRecords.map((record) => record.line).join(""));
388 onWritten(writtenRecords);
389 pendingRecords = [];
390 pendingBytes = 0;
391 });
392
393 for (const record of records) {
394 if (pendingBytes > 0 && pendingBytes + record.bytes > maxBytes) {
395 yield* flush;
396 }
397 pendingRecords.push(record);
398 pendingBytes += record.bytes;
399 if (pendingBytes >= maxBytes) {
400 yield* flush;
401 }
402 }
403 yield* flush;
404 });
405
406 async function isProviderLogFile(
407 filePath: string,
408 fileName: string,
409 filePrefix: string,
410 ): Promise<boolean> {
411 if (!/\.log(?:\.\d+)?$/u.test(fileName)) return false;
412 if (fileName.startsWith(filePrefix)) return true;
413
414 const descriptor = await NodeFS.promises.open(filePath, "r");
415 try {
416 const header = Buffer.alloc(256);
417 const { bytesRead } = await descriptor.read(header, 0, header.byteLength, 0);
418 return /^\[[^\]\r\n]+\] (?:NTIVE|CANON|ORCH): /u.test(header.toString("utf8", 0, bytesRead));
419 } finally {
420 await descriptor.close();
421 }
422 }
423
424 async function enforceRetention(input: {
425 readonly directory: string;
426 readonly maxTotalBytes: number;
427 readonly maxAgeMs: number;
428 readonly activeFilePaths: ReadonlySet<string>;
429 readonly filePrefix: string;
430 readonly now: number;
431 }): Promise<RetentionResult> {
432 const failures: Array<FileOperationFailure> = [];
433 const files: Array<{ filePath: string; mtimeMs: number; size: number }> = [];
434
435 let entries: ReadonlyArray<NodeFS.Dirent>;
436 try {
437 entries = await NodeFS.promises.readdir(input.directory, { withFileTypes: true });
438 } catch (cause) {
439 return { failures: [{ filePath: input.directory, cause }] };
440 }
441
442 for (const entry of entries) {
443 if (!entry.isFile()) continue;
444 const filePath = NodePath.join(input.directory, entry.name);
445 try {
446 if (!(await isProviderLogFile(filePath, entry.name, input.filePrefix))) continue;
447 const stat = await NodeFS.promises.stat(filePath);
448 files.push({ filePath, mtimeMs: stat.mtimeMs, size: stat.size });
449 } catch (cause) {
450 failures.push({ filePath, cause });
451 }
452 }
453
454 let totalBytes = files.reduce((total, file) => total + file.size, 0);
455 const remove = async (file: (typeof files)[number]) => {
456 if (input.activeFilePaths.has(file.filePath)) return false;
457 try {
458 await NodeFS.promises.rm(file.filePath, { force: true });
459 totalBytes -= file.size;
460 return true;
461 } catch (cause) {
462 failures.push({ filePath: file.filePath, cause });
463 return false;
464 }
465 };
466
467 const retained: typeof files = [];
468 for (const file of files) {
469 if (input.now - file.mtimeMs <= input.maxAgeMs || !(await remove(file))) retained.push(file);
470 }
471
472 for (const file of retained.toSorted(
473 (left, right) => left.mtimeMs - right.mtimeMs || left.filePath.localeCompare(right.filePath),
474 )) {
475 if (totalBytes <= input.maxTotalBytes) break;
476 await remove(file);
477 }
478
479 return { failures };
480 }
481
482 function resolveOptions(options: EventNdjsonLogStoreOptions): Effect.Effect<ResolvedOptions> {
483 const resolved = {
484 maxBytes: options.maxBytes ?? DEFAULT_MAX_BYTES,
485 maxFiles: options.maxFiles ?? DEFAULT_MAX_FILES,
486 batchWindowMs: options.batchWindowMs ?? DEFAULT_BATCH_WINDOW_MS,
487 maxTotalBytes: DEFAULT_MAX_TOTAL_BYTES,
488 maxAgeMs: DEFAULT_MAX_AGE_MS,
489 retentionCheckIntervalMs: DEFAULT_RETENTION_CHECK_INTERVAL_MS,
490 maxBufferedBytes: DEFAULT_MAX_BUFFERED_BYTES,
491 maxBufferedRecords: DEFAULT_MAX_BUFFERED_RECORDS,
492 attribution: options.attribution,
493 } satisfies ResolvedOptions;
494
495 return Effect.succeed(resolved);
496 }
497
498 const drainPending = Effect.fnUntraced(function* (input: {
499 readonly directory: string;
500 readonly options: ResolvedOptions;
501 readonly state: StoreState;
502 readonly filePrefix: string;
503 readonly now: number;
504 readonly timerFired: boolean;
505 readonly close: boolean;
506 readonly makeSink: (
507 filePath: string,
508 ) => Effect.Effect<AsyncFileSink, Effect.Error<ReturnType<typeof makeAsyncRotatingFileSink>>>;
509 }): Effect.fn.Return<readonly [DrainResult, StoreState]> {
510 if (input.state.closed) {
511 return [{ attributions: [], failures: [] }, input.state];
512 }
513
514 const sinks = new Map(input.state.sinks);
515 const writtenSegments = new Set(input.state.writtenSinceRetention);
516 const failures: Array<FileOperationFailure> = [];
517 const attributionByStream = new Map<
518 EventNdjsonStream,
519 { count: number; logicalWriteBytes: number }
520 >();
521 const recordsBySegment = new Map<string, Array<PendingRecord>>();
522
523 for (const record of input.state.pending) {
524 const records = recordsBySegment.get(record.threadSegment) ?? [];
525 records.push(record);
526 recordsBySegment.set(record.threadSegment, records);
527 }
528
529 for (const [threadSegment, records] of recordsBySegment) {
530 const filePath = providerLogPath(input.directory, input.filePrefix, threadSegment);
531 let sink = sinks.get(threadSegment);
532 if (!sink) {
533 const created = yield* input.makeSink(filePath).pipe(Effect.result);
534 if (Result.isFailure(created)) {
535 failures.push({ filePath, cause: created.failure });
536 continue;
537 }
538 sink = created.success;
539 sinks.set(threadSegment, sink);
540 }
541 writtenSegments.add(threadSegment);
542
543 const written = yield* writeBatchedMessages(
544 sink,
545 records,
546 input.options.maxBytes,
547 (writtenRecords) => {
548 for (const record of writtenRecords) {
549 const current = attributionByStream.get(record.stream) ?? {
550 count: 0,
551 logicalWriteBytes: 0,
552 };
553 attributionByStream.set(record.stream, {
554 count: current.count + 1,
555 logicalWriteBytes: current.logicalWriteBytes + record.bytes,
556 });
557 }
558 },
559 ).pipe(Effect.result);
560 if (Result.isFailure(written)) {
561 sinks.delete(threadSegment);
562 failures.push({ filePath, cause: written.failure });
563 }
564 }
565
566 const retentionDue =
567 input.now - input.state.lastRetentionAt >= input.options.retentionCheckIntervalMs;
568 if (retentionDue) {
569 // A sink idle for a whole retention interval reopens from disk on its next
570 // write, so retention may remove its files without leaving a stale size.
571 for (const threadSegment of sinks.keys()) {
572 if (!writtenSegments.has(threadSegment)) sinks.delete(threadSegment);
573 }
574 }
575 const retention = retentionDue
576 ? yield* Effect.promise(() =>
577 enforceRetention({
578 directory: input.directory,
579 maxTotalBytes: input.options.maxTotalBytes,
580 maxAgeMs: input.options.maxAgeMs,
581 activeFilePaths: new Set(
582 Array.from(sinks.keys(), (threadSegment) =>
583 providerLogPath(input.directory, input.filePrefix, threadSegment),
584 ),
585 ),
586 filePrefix: input.filePrefix,
587 now: input.now,
588 }),
589 )
590 : { failures: [] };
591
592 return [
593 {
594 attributions: Array.from(attributionByStream, ([stream, value]) => ({
595 stream,
596 ...value,
597 })),
598 failures: [...failures, ...retention.failures],
599 },
600 {
601 pending: [],
602 pendingBytes: 0,
603 sinks,
604 writtenSinceRetention: retentionDue ? new Set() : writtenSegments,
605 flushScheduled: input.timerFired ? false : input.state.flushScheduled,
606 closed: input.close,
607 lastRetentionAt: retentionDue ? input.now : input.state.lastRetentionAt,
608 },
609 ];
610 });
611
612 const serializeEvent = Effect.fnUntraced(function* (event: unknown) {
613 return yield* encodeUnknownJsonString(event).pipe(
614 Effect.catch((error) =>
615 logWarning("failed to serialize provider event log record", {
616 errorTag: errorTag(error),
617 }).pipe(Effect.as(undefined)),
618 ),
619 );
620 });
621
622 export const makeEventNdjsonLogStore = Effect.fnUntraced(function* (
623 filePath: string,
624 options: EventNdjsonLogStoreOptions = {},
625 ): Effect.fn.Return<
626 EventNdjsonLogStore,
627 EventNdjsonLogStoreError,
628 FileSystem.FileSystem | Path.Path
629 > {
630 const fs = yield* FileSystem.FileSystem;
631 const path = yield* Path.Path;
632 const resolved = yield* resolveOptions(options);
633 const directory = NodePath.dirname(filePath);
634 const filePrefix = providerLogPrefix(filePath);
635
636 yield* fs
637 .makeDirectory(directory, { recursive: true })
638 .pipe(Effect.mapError((cause) => new EventNdjsonLogDirectoryError({ directory, cause })));
639 const makeSink = (filePath: string) =>
640 makeAsyncRotatingFileSink({
641 filePath,
642 maxBytes: resolved.maxBytes,
643 maxFiles: resolved.maxFiles,
644 }).pipe(
645 Effect.provideService(FileSystem.FileSystem, fs),
646 Effect.provideService(Path.Path, path),
647 );
648
649 const initializedAt = yield* Clock.currentTimeMillis;
650 const initialRetention = yield* Effect.promise(() =>
651 enforceRetention({
652 directory,
653 maxTotalBytes: resolved.maxTotalBytes,
654 maxAgeMs: resolved.maxAgeMs,
655 activeFilePaths: new Set(),
656 filePrefix,
657 now: initializedAt,
658 }),
659 );
660 for (const failure of initialRetention.failures) {
661 yield* logWarning("provider event log retention failed", {
662 filePath: failure.filePath,
663 errorTag: errorTag(failure.cause),
664 });
665 }
666
667 const stateRef = yield* SynchronizedRef.make<StoreState>({
668 pending: [],
669 pendingBytes: 0,
670 sinks: new Map(),
671 writtenSinceRetention: new Set(),
672 flushScheduled: false,
673 closed: false,
674 lastRetentionAt: initializedAt,
675 });
676 const timerScope = yield* Scope.make();
677
678 const flush = Effect.fnUntraced(function* (timerFired: boolean, close: boolean) {
679 const startedAt = yield* Clock.currentTimeMillis;
680 const result = yield* SynchronizedRef.modifyEffect(stateRef, (state) =>
681 drainPending({
682 directory,
683 options: resolved,
684 state,
685 filePrefix,
686 now: startedAt,
687 timerFired,
688 close,
689 makeSink,
690 }),
691 );
692
693 for (const failure of result.failures) {
694 yield* logWarning("provider event log write or retention failed", {
695 filePath: failure.filePath,
696 errorTag: errorTag(failure.cause),
697 });
698 }
699
700 if (resolved.attribution && result.attributions.length > 0) {
701 const completedAt = yield* Clock.currentTimeMillis;
702 const durationMs = Math.max(0, completedAt - startedAt);
703 const totalBytes = result.attributions.reduce(
704 (total, entry) => total + entry.logicalWriteBytes,
705 0,
706 );
707 yield* Effect.forEach(
708 result.attributions,
709 (entry) =>
710 resolved.attribution?.record({
711 component: "provider-event-log",
712 operation: `${entry.stream}.append`,
713 logicalWriteBytes: entry.logicalWriteBytes,
714 count: entry.count,
715 durationMs:
716 totalBytes === 0
717 ? 0
718 : Math.round(durationMs * (entry.logicalWriteBytes / totalBytes)),
719 }) ?? Effect.void,
720 { discard: true },
721 );
722 }
723 }, Effect.uninterruptible);
724
725 const scheduleFlush = Effect.fnUntraced(function* () {
726 yield* Effect.forkIn(
727 Effect.sleep(resolved.batchWindowMs).pipe(Effect.andThen(flush(true, false))),
728 timerScope,
729 { startImmediately: true },
730 );
731 });
732
733 const close = Effect.fnUntraced(function* () {
734 yield* flush(false, true);
735 yield* Scope.close(timerScope, Exit.void);
736 });
737
738 const loggerViews = new Map<EventNdjsonStream, ProviderEventLoggers.EventNdjsonLogger>();
739 const logger = (stream: EventNdjsonStream): ProviderEventLoggers.EventNdjsonLogger => {
740 const existing = loggerViews.get(stream);
741 if (existing) return existing;
742
743 const write = Effect.fnUntraced(function* (event: unknown, threadId: ThreadId | null) {
744 if (!shouldPersistProviderEvent(stream, event)) return;
745 let payload = yield* serializeEvent(boundProviderEventForLogging(event));
746 if (payload === undefined) return;
747 // Escaping can expand strings beyond their input size.
748 if (Buffer.byteLength(payload) > MAX_RECORD_CHARACTERS) {
749 payload = yield* serializeEvent(summarizeProviderEvent(event));
750 if (payload === undefined) return;
751 }
752
753 const observedAt = yield* DateTime.now.pipe(Effect.map(DateTime.formatIso));
754 const line = `[${observedAt}] ${resolveStreamLabel(stream)}: ${payload}\n`;
755 const bytes = Buffer.byteLength(line);
756 const threadSegment = resolveThreadSegment(threadId);
757 const action = yield* SynchronizedRef.modifyEffect(stateRef, (state) => {
758 if (state.closed) {
759 return Effect.succeed([{ flush: false }, state] as const);
760 }
761 const pending = state.pending;
762 pending.push({ stream, threadSegment, line, bytes });
763 const pendingBytes = state.pendingBytes + bytes;
764 const flush =
765 resolved.batchWindowMs === 0 ||
766 pending.length >= resolved.maxBufferedRecords ||
767 pendingBytes >= resolved.maxBufferedBytes;
768 const schedule = !flush && !state.flushScheduled;
769 const nextState = {
770 ...state,
771 pending,
772 pendingBytes,
773 flushScheduled: state.flushScheduled || schedule,
774 };
775 return (schedule ? scheduleFlush() : Effect.void).pipe(
776 Effect.as([{ flush }, nextState] as const),
777 );
778 }).pipe(Effect.uninterruptible);
779
780 if (action.flush) {
781 yield* flush(false, false);
782 }
783 });
784
785 const view = {
786 filePath,
787 write,
788 close: () => Effect.void,
789 } satisfies ProviderEventLoggers.EventNdjsonLogger;
790 loggerViews.set(stream, view);
791 return view;
792 };
793
794 return { filePath, logger, close } satisfies EventNdjsonLogStore;
795 });