The best way to control Claude Code, Codex and more from any computer or phone

main
ts 611 lines 22.8 KB
Raw
1 // @effect-diagnostics nodeBuiltinImport:off
2 import * as NodeFS from "node:fs";
3 import * as NodeOS from "node:os";
4 import * as NodePath from "node:path";
5
6 import { ThreadId } from "@t3tools/contracts";
7 import * as NodeServices from "@effect/platform-node/NodeServices";
8 import { assert, it } from "@effect/vitest";
9 import { RotatingFileSinkError } from "@t3tools/shared/logging";
10 import * as Effect from "effect/Effect";
11 import * as Deferred from "effect/Deferred";
12 import * as FileSystem from "effect/FileSystem";
13 import * as Fiber from "effect/Fiber";
14 import * as Logger from "effect/Logger";
15 import * as Schema from "effect/Schema";
16 import * as TestClock from "effect/testing/TestClock";
17
18 import {
19 makeEventNdjsonLogStore,
20 type PendingRecord,
21 writeBatchedMessages,
22 } from "./EventNdjsonLogger.ts";
23
24 const encodeUnknownJson = Schema.encodeUnknownSync(Schema.fromJsonString(Schema.Unknown));
25 const decodeUnknownJson = Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown));
26
27 function ownedLogPath(basePath: string, segment: string): string {
28 const basename = NodePath.basename(basePath);
29 const extension = NodePath.extname(basename);
30 const stem = extension.length > 0 ? basename.slice(0, -extension.length) : basename;
31 return NodePath.join(NodePath.dirname(basePath), `${stem}.${segment}.log`);
32 }
33
34 function parseLogLine(line: string) {
35 const match = /^\[([^\]]+)\] ([A-Z]+): (.+)$/.exec(line);
36 assert.notEqual(match, null);
37 if (!match) {
38 throw new Error(`invalid log line: ${line}`);
39 }
40 const observedAt = match[1];
41 const stream = match[2];
42 const payload = match[3];
43 if (!observedAt || !stream || payload === undefined) {
44 throw new Error(`invalid log line: ${line}`);
45 }
46 return {
47 observedAt,
48 stream,
49 payload,
50 };
51 }
52
53 it.layer(NodeServices.layer)("EventNdjsonLogger", (it) => {
54 it.effect("summarizes circular events without exposing their contents in diagnostics", () => {
55 const messages: Array<unknown> = [];
56 const logCapture = Logger.make<unknown, void>(({ message }) => {
57 if (Array.isArray(message)) {
58 messages.push(...message);
59 } else {
60 messages.push(message);
61 }
62 });
63 const secret = "secret-circular-event-value";
64
65 return Effect.gen(function* () {
66 const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-"));
67 const basePath = NodePath.join(tempDir, "provider-native.ndjson");
68 const circular: Record<string, unknown> = { secret };
69 circular.self = circular;
70
71 try {
72 const store = yield* makeEventNdjsonLogStore(basePath);
73 const logger = store.logger("native");
74 yield* logger.write(circular, ThreadId.make("thread-1"));
75 yield* store.close();
76
77 const serialized = encodeUnknownJson(messages);
78 assert.notInclude(serialized, secret);
79 const line = parseLogLine(
80 NodeFS.readFileSync(ownedLogPath(basePath, "thread-1"), "utf8").trim(),
81 );
82 assert.equal(line.payload, '{"truncated":true}');
83 } finally {
84 NodeFS.rmSync(tempDir, { recursive: true, force: true });
85 }
86 }).pipe(Effect.provide(Logger.layer([logCapture], { mergeWithExisting: false })));
87 });
88
89 it.effect(
90 "falls back to a global segment when orchestration thread id is missing or invalid",
91 () =>
92 Effect.gen(function* () {
93 const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-"));
94 const basePath = NodePath.join(tempDir, "provider-canonical.ndjson");
95
96 try {
97 const store = yield* makeEventNdjsonLogStore(basePath);
98 const logger = store.logger("orchestration");
99
100 yield* logger.write({ id: "evt-no-thread" }, null);
101 yield* logger.write({ id: "evt-invalid-thread" }, "!!!" as unknown as ThreadId);
102 yield* store.close();
103
104 const globalPath = ownedLogPath(basePath, "_global");
105 assert.equal(NodeFS.existsSync(globalPath), true);
106 const lines = NodeFS.readFileSync(globalPath, "utf8")
107 .trim()
108 .split("\n")
109 .map((line) => parseLogLine(line));
110 assert.equal(lines.length, 2);
111 assert.equal(Number.isNaN(Date.parse(lines[0]?.observedAt ?? "")), false);
112 assert.equal(Number.isNaN(Date.parse(lines[1]?.observedAt ?? "")), false);
113 assert.equal(lines[0]?.stream, "ORCH");
114 assert.equal(lines[0]?.payload, '{"id":"evt-no-thread"}');
115 assert.equal(lines[1]?.stream, "ORCH");
116 assert.equal(lines[1]?.payload, '{"id":"evt-invalid-thread"}');
117 } finally {
118 NodeFS.rmSync(tempDir, { recursive: true, force: true });
119 }
120 }),
121 );
122
123 it.effect("keeps long thread ids that share a prefix in separate files", () =>
124 Effect.gen(function* () {
125 const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-"));
126 const basePath = NodePath.join(tempDir, "events.log");
127 const prefix =
128 "thread:delegated-task:command%3Amcp%3A00000000-0000-4000-8000-000000000000%3Adelegate-task%3A";
129
130 try {
131 const store = yield* makeEventNdjsonLogStore(basePath, { batchWindowMs: 0 });
132 const logger = store.logger("native");
133 yield* logger.write({ id: "first" }, ThreadId.make(`${prefix}review-the-diff`));
134 yield* logger.write({ id: "second" }, ThreadId.make(`${prefix}write-the-tests`));
135 yield* store.close();
136
137 const files = NodeFS.readdirSync(tempDir);
138 assert.equal(files.length, 2);
139 assert.deepEqual(
140 files
141 .map((name) =>
142 NodeFS.readFileSync(NodePath.join(tempDir, name), "utf8")
143 .trim()
144 .split("\n")
145 .map((line) => parseLogLine(line).payload),
146 )
147 .toSorted(),
148 [['{"id":"first"}'], ['{"id":"second"}']],
149 );
150 } finally {
151 NodeFS.rmSync(tempDir, { recursive: true, force: true });
152 }
153 }),
154 );
155
156 it.effect("keeps shared store views non-owning when one adapter closes", () =>
157 Effect.gen(function* () {
158 const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-"));
159 const basePath = NodePath.join(tempDir, "events.log");
160
161 try {
162 const store = yield* makeEventNdjsonLogStore(basePath, { batchWindowMs: 0 });
163 const native = store.logger("native");
164 const canonical = store.logger("canonical");
165 const threadId = ThreadId.make("thread-shared-close");
166
167 yield* native.write({ id: "before-close" }, threadId);
168 yield* native.close();
169 yield* canonical.write({ type: "item.completed", id: "after-close" }, threadId);
170 yield* store.close();
171
172 const lines = NodeFS.readFileSync(ownedLogPath(basePath, "thread-shared-close"), "utf8")
173 .trim()
174 .split("\n")
175 .map(parseLogLine);
176
177 assert.deepEqual(
178 lines.map(({ stream, payload }) => ({ stream, payload })),
179 [
180 { stream: "NTIVE", payload: '{"id":"before-close"}' },
181 {
182 stream: "CANON",
183 payload: '{"type":"item.completed","id":"after-close"}',
184 },
185 ],
186 );
187 } finally {
188 NodeFS.rmSync(tempDir, { recursive: true, force: true });
189 }
190 }),
191 );
192
193 it.effect("does not strand a later batch after an interrupted write", () =>
194 Effect.gen(function* () {
195 const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-"));
196 const basePath = NodePath.join(tempDir, "events.log");
197 const threadPath = ownedLogPath(basePath, "thread-interrupted");
198
199 try {
200 const store = yield* makeEventNdjsonLogStore(basePath, { batchWindowMs: 1_000 });
201 const logger = store.logger("native");
202 const interruptedWrite = yield* logger
203 .write({ id: "possibly-interrupted" }, ThreadId.make("thread-interrupted"))
204 .pipe(Effect.forkChild);
205 yield* Effect.yieldNow;
206 yield* Fiber.interrupt(interruptedWrite);
207 yield* logger.write({ id: "accepted" }, ThreadId.make("thread-interrupted"));
208
209 yield* TestClock.adjust(1_000);
210 yield* store.close();
211
212 assert.equal(NodeFS.existsSync(threadPath), true);
213 assert.include(NodeFS.readFileSync(threadPath, "utf8"), '{"id":"accepted"}');
214 } finally {
215 NodeFS.rmSync(tempDir, { recursive: true, force: true });
216 }
217 }),
218 );
219
220 it.effect("summarizes large histories without reading their items", () =>
221 Effect.gen(function* () {
222 const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-"));
223 const basePath = NodePath.join(tempDir, "events.log");
224 const turns = Array.from({ length: 10_000 });
225 Object.defineProperty(turns, 0, {
226 get: () => {
227 throw new Error("history must not be serialized");
228 },
229 });
230
231 try {
232 const store = yield* makeEventNdjsonLogStore(basePath);
233 yield* store.logger("native").write(
234 {
235 provider: "codex",
236 event: {
237 direction: "incoming",
238 stage: "decoded",
239 payload: { id: 42, result: { thread: { id: "native-thread", turns } } },
240 },
241 },
242 ThreadId.make("large-history"),
243 );
244 yield* store.close();
245
246 const contents = NodeFS.readFileSync(ownedLogPath(basePath, "large-history"), "utf8");
247 assert.isBelow(Buffer.byteLength(contents), 2_048);
248 const record = decodeUnknownJson(parseLogLine(contents.trim()).payload);
249 assert.nestedPropertyVal(record, "event.payload.id", 42);
250 assert.nestedPropertyVal(record, "event.payload.result.thread.id", "native-thread");
251 assert.nestedPropertyVal(record, "event.payload.result.thread.turns.itemCount", 10_000);
252 } finally {
253 NodeFS.rmSync(tempDir, { recursive: true, force: true });
254 }
255 }),
256 );
257
258 it.effect("drops Cursor streaming deltas while keeping its lifecycle updates", () =>
259 Effect.gen(function* () {
260 const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-"));
261 const basePath = NodePath.join(tempDir, "events.log");
262 const threadId = ThreadId.make("cursor-deltas");
263 const cursorUpdate = (update: object) => ({
264 provider: "cursor",
265 kind: "protocol",
266 event: {
267 direction: "incoming",
268 stage: "decoded",
269 payload: { type: "interaction.update", runId: "run-1", update },
270 },
271 });
272 const toolCall = { type: "shell", args: { command: "ls" } };
273 const transient = [
274 { type: "text-delta", text: "Hel" },
275 { type: "thinking-delta", text: "Hmm" },
276 { type: "token-delta", tokens: 3 },
277 { type: "shell-output-delta", event: { stdout: "a.ts\n" } },
278 { type: "partial-tool-call", callId: "call-1", modelCallId: "m-1", toolCall },
279 {
280 type: "tool-call-delta",
281 callId: "call-1",
282 modelCallId: "m-1",
283 taskUpdate: { type: "text-delta", text: "Sub" },
284 },
285 ].map(cursorUpdate);
286 const retained = [
287 { type: "tool-call-started", callId: "call-1", modelCallId: "m-1", toolCall },
288 {
289 type: "tool-call-delta",
290 callId: "call-1",
291 modelCallId: "m-1",
292 taskUpdate: { type: "tool-call-completed", callId: "call-2", toolCall },
293 },
294 { type: "tool-call-completed", callId: "call-1", modelCallId: "m-1", toolCall },
295 { type: "turn-ended" },
296 ].map(cursorUpdate);
297
298 try {
299 const store = yield* makeEventNdjsonLogStore(basePath, { batchWindowMs: 0 });
300 const native = store.logger("native");
301 for (const event of [...transient, ...retained]) {
302 yield* native.write(event, threadId);
303 }
304 yield* store.close();
305
306 const records = NodeFS.readFileSync(ownedLogPath(basePath, "cursor-deltas"), "utf8")
307 .trim()
308 .split("\n")
309 .map((line) => decodeUnknownJson(parseLogLine(line).payload));
310 assert.deepEqual(records, retained);
311 } finally {
312 NodeFS.rmSync(tempDir, { recursive: true, force: true });
313 }
314 }),
315 );
316
317 it.effect("bounds oversized records while retaining failure details", () =>
318 Effect.gen(function* () {
319 const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-"));
320 const basePath = NodePath.join(tempDir, "events.log");
321
322 try {
323 const store = yield* makeEventNdjsonLogStore(basePath);
324 const logger = store.logger("native");
325 const threadId = ThreadId.make("large-error");
326 const failure = {
327 method: "error",
328 params: {
329 threadId: "native-thread",
330 turnId: "native-turn",
331 error: { message: "The provider is unavailable.", code: "overloaded" },
332 output: "x".repeat(128 * 1_024),
333 },
334 };
335 yield* logger.write(failure, threadId);
336 yield* logger.write({ id: "escaped", output: "\u0000".repeat(20_000) }, threadId);
337 yield* store.close();
338
339 const contents = NodeFS.readFileSync(ownedLogPath(basePath, "large-error"), "utf8");
340 const records = contents
341 .trim()
342 .split("\n")
343 .map((line) => decodeUnknownJson(parseLogLine(line).payload));
344 assert.isBelow(Buffer.byteLength(contents), 64 * 1_024);
345 assert.equal(records.length, 2);
346 assert.nestedPropertyVal(records[0], "params.threadId", "native-thread");
347 assert.nestedPropertyVal(records[0], "params.turnId", "native-turn");
348 assert.nestedPropertyVal(
349 records[0],
350 "params.error.message",
351 "The provider is unavailable.",
352 );
353 assert.nestedPropertyVal(records[0], "params.error.code", "overloaded");
354 assert.propertyVal(records[1], "id", "escaped");
355 } finally {
356 NodeFS.rmSync(tempDir, { recursive: true, force: true });
357 }
358 }),
359 );
360
361 it.effect("bounds canonical diff snapshots before serializing their duplicate payloads", () =>
362 Effect.gen(function* () {
363 const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-"));
364 const basePath = NodePath.join(tempDir, "events.log");
365 const threadId = ThreadId.make("large-diff");
366 const diff = "diff-payload".repeat(128 * 1_024);
367 try {
368 const store = yield* makeEventNdjsonLogStore(basePath, { batchWindowMs: 0 });
369 yield* store.logger("canonical").write(
370 {
371 type: "turn.diff.updated",
372 threadId,
373 turnId: "native-turn",
374 raw: { method: "turn/diff/updated", payload: { diff } },
375 payload: { unifiedDiff: diff },
376 },
377 threadId,
378 );
379 yield* store.close();
380 const contents = NodeFS.readFileSync(ownedLogPath(basePath, "large-diff"), "utf8");
381 assert.isBelow(Buffer.byteLength(contents), 2_048);
382 const record = decodeUnknownJson(parseLogLine(contents.trim()).payload);
383 assert.propertyVal(record, "type", "turn.diff.updated");
384 assert.propertyVal(record, "threadId", threadId);
385 assert.propertyVal(record, "turnId", "native-turn");
386 } finally {
387 NodeFS.rmSync(tempDir, { recursive: true, force: true });
388 }
389 }),
390 );
391
392 it.effect("serializes concurrent first writes for the same segment", () =>
393 Effect.gen(function* () {
394 const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-"));
395 const basePath = NodePath.join(tempDir, "provider-canonical.ndjson");
396
397 try {
398 const store = yield* makeEventNdjsonLogStore(basePath, { batchWindowMs: 0 });
399 const logger = store.logger("canonical");
400
401 yield* Effect.all(
402 [
403 logger.write({ id: "evt-concurrent-1" }, null),
404 logger.write({ id: "evt-concurrent-2" }, null),
405 ],
406 { concurrency: "unbounded" },
407 );
408 yield* store.close();
409
410 const globalPath = ownedLogPath(basePath, "_global");
411 assert.equal(NodeFS.existsSync(globalPath), true);
412 const lines = NodeFS.readFileSync(globalPath, "utf8")
413 .trim()
414 .split("\n")
415 .map((line) => parseLogLine(line));
416
417 assert.equal(lines.length, 2);
418 assert.deepEqual(lines.map((line) => line.payload).toSorted(), [
419 '{"id":"evt-concurrent-1"}',
420 '{"id":"evt-concurrent-2"}',
421 ]);
422 } finally {
423 NodeFS.rmSync(tempDir, { recursive: true, force: true });
424 }
425 }),
426 );
427
428 it.effect("rotates per-thread files when max size is exceeded", () =>
429 Effect.gen(function* () {
430 const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-"));
431 const basePath = NodePath.join(tempDir, "provider-native.ndjson");
432
433 try {
434 const store = yield* makeEventNdjsonLogStore(basePath, {
435 maxBytes: 120,
436 maxFiles: 2,
437 batchWindowMs: 0,
438 });
439 const native = store.logger("native");
440 const canonical = store.logger("canonical");
441
442 for (let index = 0; index < 10; index += 1) {
443 const logger = index % 2 === 0 ? native : canonical;
444 yield* logger.write(
445 {
446 type: "session.started",
447 threadId: "provider-thread-rotate",
448 id: `evt-${index}`,
449 payload: "x".repeat(40),
450 },
451 ThreadId.make("thread-rotate"),
452 );
453 }
454 yield* store.close();
455
456 const fileStem = NodePath.basename(ownedLogPath(basePath, "thread-rotate"));
457 const matchingFiles = NodeFS.readdirSync(tempDir)
458 .filter((entry) => entry === fileStem || entry.startsWith(`${fileStem}.`))
459 .toSorted();
460
461 assert.equal(
462 matchingFiles.some((entry) => entry === `${fileStem}.1`),
463 true,
464 );
465 assert.equal(
466 matchingFiles.some((entry) => entry === fileStem || entry === `${fileStem}.2`),
467 true,
468 );
469 assert.equal(
470 matchingFiles.some((entry) => entry === `${fileStem}.3`),
471 false,
472 );
473 } finally {
474 NodeFS.rmSync(tempDir, { recursive: true, force: true });
475 }
476 }),
477 );
478
479 it.effect("retires a thread log once a retention interval passes without writes", () =>
480 Effect.gen(function* () {
481 const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-"));
482 const basePath = NodePath.join(tempDir, "events.log");
483 const finishedPath = ownedLogPath(basePath, "finished");
484 const activePath = ownedLogPath(basePath, "active");
485
486 try {
487 const store = yield* makeEventNdjsonLogStore(basePath, {
488 batchWindowMs: 0,
489 });
490 const logger = store.logger("native");
491
492 yield* logger.write({ id: "finished-first" }, ThreadId.make("finished"));
493 yield* TestClock.adjust("5 minutes");
494 yield* logger.write({ id: "active-first" }, ThreadId.make("active"));
495 assert.equal(NodeFS.existsSync(finishedPath), true);
496 NodeFS.truncateSync(finishedPath, 512 * 1024 * 1024 + 1);
497
498 yield* TestClock.adjust("5 minutes");
499 yield* logger.write({ id: "active-second" }, ThreadId.make("active"));
500 assert.equal(NodeFS.existsSync(finishedPath), false);
501 assert.equal(NodeFS.existsSync(activePath), true);
502
503 yield* logger.write({ id: "finished-again" }, ThreadId.make("finished"));
504 yield* store.close();
505 assert.deepEqual(
506 NodeFS.readFileSync(finishedPath, "utf8")
507 .trim()
508 .split("\n")
509 .map((line) => parseLogLine(line).payload),
510 ['{"id":"finished-again"}'],
511 );
512 } finally {
513 NodeFS.rmSync(tempDir, { recursive: true, force: true });
514 }
515 }),
516 );
517
518 it.effect("attributes batches that were written before a later chunk fails", () =>
519 Effect.gen(function* () {
520 const records: ReadonlyArray<PendingRecord> = [
521 {
522 stream: "native",
523 threadSegment: "thread",
524 line: "first",
525 bytes: 5,
526 },
527 {
528 stream: "canonical",
529 threadSegment: "thread",
530 line: "second",
531 bytes: 6,
532 },
533 ];
534 const attributed: Array<PendingRecord> = [];
535 let writes = 0;
536
537 const result = yield* writeBatchedMessages(
538 {
539 write: () => {
540 writes += 1;
541 return writes === 2
542 ? Effect.fail(
543 new RotatingFileSinkError({
544 operation: "write",
545 filePath: "test.log",
546 cause: "simulated disk exhaustion",
547 }),
548 )
549 : Effect.void;
550 },
551 },
552 records,
553 5,
554 (written) => attributed.push(...written),
555 ).pipe(Effect.result);
556 assert.equal(result._tag, "Failure");
557 assert.deepEqual(attributed, [records[0]]);
558 }),
559 );
560
561 it.effect(
562 "keeps unrelated work responsive and drains before close while a disk write is held",
563 () =>
564 Effect.gen(function* () {
565 const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-"));
566 const basePath = NodePath.join(tempDir, "events.log");
567 const fs = yield* FileSystem.FileSystem;
568 const started = yield* Deferred.make<void>();
569 const release = yield* Deferred.make<void>();
570 let writes = 0;
571 try {
572 const store = yield* makeEventNdjsonLogStore(basePath, { batchWindowMs: 0 }).pipe(
573 Effect.provideService(FileSystem.FileSystem, {
574 ...fs,
575 writeFile: (filePath, content, options) =>
576 Effect.gen(function* () {
577 if (++writes === 1) {
578 yield* Deferred.succeed(started, undefined);
579 yield* Deferred.await(release);
580 }
581 yield* fs.writeFile(filePath, content, options);
582 }),
583 }),
584 );
585 const first = yield* store
586 .logger("native")
587 .write({ id: "first" }, ThreadId.make("held"))
588 .pipe(Effect.forkChild);
589 yield* Deferred.await(started);
590 const closed = yield* Deferred.make<void>();
591 const close = yield* store
592 .close()
593 .pipe(Effect.andThen(Deferred.succeed(closed, undefined)), Effect.forkChild);
594 assert.equal(yield* Deferred.isDone(closed), false);
595
596 const heartbeat = yield* Deferred.make<void>();
597 yield* Effect.forkChild(Deferred.succeed(heartbeat, undefined));
598 yield* Deferred.await(heartbeat);
599 yield* Deferred.succeed(release, undefined);
600 yield* Fiber.join(first);
601 yield* Fiber.join(close);
602 assert.include(
603 NodeFS.readFileSync(ownedLogPath(basePath, "held"), "utf8"),
604 '{"id":"first"}',
605 );
606 } finally {
607 NodeFS.rmSync(tempDir, { recursive: true, force: true });
608 }
609 }),
610 );
611 });