The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 04:45 UTC
| 1 | // @effect-diagnostics nodeBuiltinImport:off |
| 2 | import * as NodeFS from "node:fs"; |
| 3 | import * as NodeOS from "node:os"; |
| 4 | import * as NodePath from "node:path"; |
| 5 | |
| 6 | import { ThreadId } from "@t3tools/contracts"; |
| 7 | import * as NodeServices from "@effect/platform-node/NodeServices"; |
| 8 | import { assert, it } from "@effect/vitest"; |
| 9 | import { RotatingFileSinkError } from "@t3tools/shared/logging"; |
| 10 | import * as Effect from "effect/Effect"; |
| 11 | import * as Deferred from "effect/Deferred"; |
| 12 | import * as FileSystem from "effect/FileSystem"; |
| 13 | import * as Fiber from "effect/Fiber"; |
| 14 | import * as Logger from "effect/Logger"; |
| 15 | import * as Schema from "effect/Schema"; |
| 16 | import * as TestClock from "effect/testing/TestClock"; |
| 17 | |
| 18 | import { |
| 19 | makeEventNdjsonLogStore, |
| 20 | type PendingRecord, |
| 21 | writeBatchedMessages, |
| 22 | } from "./EventNdjsonLogger.ts"; |
| 23 | |
| 24 | const encodeUnknownJson = Schema.encodeUnknownSync(Schema.fromJsonString(Schema.Unknown)); |
| 25 | const decodeUnknownJson = Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown)); |
| 26 | |
| 27 | function ownedLogPath(basePath: string, segment: string): string { |
| 28 | const basename = NodePath.basename(basePath); |
| 29 | const extension = NodePath.extname(basename); |
| 30 | const stem = extension.length > 0 ? basename.slice(0, -extension.length) : basename; |
| 31 | return NodePath.join(NodePath.dirname(basePath), `${stem}.${segment}.log`); |
| 32 | } |
| 33 | |
| 34 | function parseLogLine(line: string) { |
| 35 | const match = /^\[([^\]]+)\] ([A-Z]+): (.+)$/.exec(line); |
| 36 | assert.notEqual(match, null); |
| 37 | if (!match) { |
| 38 | throw new Error(`invalid log line: ${line}`); |
| 39 | } |
| 40 | const observedAt = match[1]; |
| 41 | const stream = match[2]; |
| 42 | const payload = match[3]; |
| 43 | if (!observedAt || !stream || payload === undefined) { |
| 44 | throw new Error(`invalid log line: ${line}`); |
| 45 | } |
| 46 | return { |
| 47 | observedAt, |
| 48 | stream, |
| 49 | payload, |
| 50 | }; |
| 51 | } |
| 52 | |
| 53 | it.layer(NodeServices.layer)("EventNdjsonLogger", (it) => { |
| 54 | it.effect("summarizes circular events without exposing their contents in diagnostics", () => { |
| 55 | const messages: Array<unknown> = []; |
| 56 | const logCapture = Logger.make<unknown, void>(({ message }) => { |
| 57 | if (Array.isArray(message)) { |
| 58 | messages.push(...message); |
| 59 | } else { |
| 60 | messages.push(message); |
| 61 | } |
| 62 | }); |
| 63 | const secret = "secret-circular-event-value"; |
| 64 | |
| 65 | return Effect.gen(function* () { |
| 66 | const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-")); |
| 67 | const basePath = NodePath.join(tempDir, "provider-native.ndjson"); |
| 68 | const circular: Record<string, unknown> = { secret }; |
| 69 | circular.self = circular; |
| 70 | |
| 71 | try { |
| 72 | const store = yield* makeEventNdjsonLogStore(basePath); |
| 73 | const logger = store.logger("native"); |
| 74 | yield* logger.write(circular, ThreadId.make("thread-1")); |
| 75 | yield* store.close(); |
| 76 | |
| 77 | const serialized = encodeUnknownJson(messages); |
| 78 | assert.notInclude(serialized, secret); |
| 79 | const line = parseLogLine( |
| 80 | NodeFS.readFileSync(ownedLogPath(basePath, "thread-1"), "utf8").trim(), |
| 81 | ); |
| 82 | assert.equal(line.payload, '{"truncated":true}'); |
| 83 | } finally { |
| 84 | NodeFS.rmSync(tempDir, { recursive: true, force: true }); |
| 85 | } |
| 86 | }).pipe(Effect.provide(Logger.layer([logCapture], { mergeWithExisting: false }))); |
| 87 | }); |
| 88 | |
| 89 | it.effect( |
| 90 | "falls back to a global segment when orchestration thread id is missing or invalid", |
| 91 | () => |
| 92 | Effect.gen(function* () { |
| 93 | const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-")); |
| 94 | const basePath = NodePath.join(tempDir, "provider-canonical.ndjson"); |
| 95 | |
| 96 | try { |
| 97 | const store = yield* makeEventNdjsonLogStore(basePath); |
| 98 | const logger = store.logger("orchestration"); |
| 99 | |
| 100 | yield* logger.write({ id: "evt-no-thread" }, null); |
| 101 | yield* logger.write({ id: "evt-invalid-thread" }, "!!!" as unknown as ThreadId); |
| 102 | yield* store.close(); |
| 103 | |
| 104 | const globalPath = ownedLogPath(basePath, "_global"); |
| 105 | assert.equal(NodeFS.existsSync(globalPath), true); |
| 106 | const lines = NodeFS.readFileSync(globalPath, "utf8") |
| 107 | .trim() |
| 108 | .split("\n") |
| 109 | .map((line) => parseLogLine(line)); |
| 110 | assert.equal(lines.length, 2); |
| 111 | assert.equal(Number.isNaN(Date.parse(lines[0]?.observedAt ?? "")), false); |
| 112 | assert.equal(Number.isNaN(Date.parse(lines[1]?.observedAt ?? "")), false); |
| 113 | assert.equal(lines[0]?.stream, "ORCH"); |
| 114 | assert.equal(lines[0]?.payload, '{"id":"evt-no-thread"}'); |
| 115 | assert.equal(lines[1]?.stream, "ORCH"); |
| 116 | assert.equal(lines[1]?.payload, '{"id":"evt-invalid-thread"}'); |
| 117 | } finally { |
| 118 | NodeFS.rmSync(tempDir, { recursive: true, force: true }); |
| 119 | } |
| 120 | }), |
| 121 | ); |
| 122 | |
| 123 | it.effect("keeps long thread ids that share a prefix in separate files", () => |
| 124 | Effect.gen(function* () { |
| 125 | const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-")); |
| 126 | const basePath = NodePath.join(tempDir, "events.log"); |
| 127 | const prefix = |
| 128 | "thread:delegated-task:command%3Amcp%3A00000000-0000-4000-8000-000000000000%3Adelegate-task%3A"; |
| 129 | |
| 130 | try { |
| 131 | const store = yield* makeEventNdjsonLogStore(basePath, { batchWindowMs: 0 }); |
| 132 | const logger = store.logger("native"); |
| 133 | yield* logger.write({ id: "first" }, ThreadId.make(`${prefix}review-the-diff`)); |
| 134 | yield* logger.write({ id: "second" }, ThreadId.make(`${prefix}write-the-tests`)); |
| 135 | yield* store.close(); |
| 136 | |
| 137 | const files = NodeFS.readdirSync(tempDir); |
| 138 | assert.equal(files.length, 2); |
| 139 | assert.deepEqual( |
| 140 | files |
| 141 | .map((name) => |
| 142 | NodeFS.readFileSync(NodePath.join(tempDir, name), "utf8") |
| 143 | .trim() |
| 144 | .split("\n") |
| 145 | .map((line) => parseLogLine(line).payload), |
| 146 | ) |
| 147 | .toSorted(), |
| 148 | [['{"id":"first"}'], ['{"id":"second"}']], |
| 149 | ); |
| 150 | } finally { |
| 151 | NodeFS.rmSync(tempDir, { recursive: true, force: true }); |
| 152 | } |
| 153 | }), |
| 154 | ); |
| 155 | |
| 156 | it.effect("keeps shared store views non-owning when one adapter closes", () => |
| 157 | Effect.gen(function* () { |
| 158 | const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-")); |
| 159 | const basePath = NodePath.join(tempDir, "events.log"); |
| 160 | |
| 161 | try { |
| 162 | const store = yield* makeEventNdjsonLogStore(basePath, { batchWindowMs: 0 }); |
| 163 | const native = store.logger("native"); |
| 164 | const canonical = store.logger("canonical"); |
| 165 | const threadId = ThreadId.make("thread-shared-close"); |
| 166 | |
| 167 | yield* native.write({ id: "before-close" }, threadId); |
| 168 | yield* native.close(); |
| 169 | yield* canonical.write({ type: "item.completed", id: "after-close" }, threadId); |
| 170 | yield* store.close(); |
| 171 | |
| 172 | const lines = NodeFS.readFileSync(ownedLogPath(basePath, "thread-shared-close"), "utf8") |
| 173 | .trim() |
| 174 | .split("\n") |
| 175 | .map(parseLogLine); |
| 176 | |
| 177 | assert.deepEqual( |
| 178 | lines.map(({ stream, payload }) => ({ stream, payload })), |
| 179 | [ |
| 180 | { stream: "NTIVE", payload: '{"id":"before-close"}' }, |
| 181 | { |
| 182 | stream: "CANON", |
| 183 | payload: '{"type":"item.completed","id":"after-close"}', |
| 184 | }, |
| 185 | ], |
| 186 | ); |
| 187 | } finally { |
| 188 | NodeFS.rmSync(tempDir, { recursive: true, force: true }); |
| 189 | } |
| 190 | }), |
| 191 | ); |
| 192 | |
| 193 | it.effect("does not strand a later batch after an interrupted write", () => |
| 194 | Effect.gen(function* () { |
| 195 | const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-")); |
| 196 | const basePath = NodePath.join(tempDir, "events.log"); |
| 197 | const threadPath = ownedLogPath(basePath, "thread-interrupted"); |
| 198 | |
| 199 | try { |
| 200 | const store = yield* makeEventNdjsonLogStore(basePath, { batchWindowMs: 1_000 }); |
| 201 | const logger = store.logger("native"); |
| 202 | const interruptedWrite = yield* logger |
| 203 | .write({ id: "possibly-interrupted" }, ThreadId.make("thread-interrupted")) |
| 204 | .pipe(Effect.forkChild); |
| 205 | yield* Effect.yieldNow; |
| 206 | yield* Fiber.interrupt(interruptedWrite); |
| 207 | yield* logger.write({ id: "accepted" }, ThreadId.make("thread-interrupted")); |
| 208 | |
| 209 | yield* TestClock.adjust(1_000); |
| 210 | yield* store.close(); |
| 211 | |
| 212 | assert.equal(NodeFS.existsSync(threadPath), true); |
| 213 | assert.include(NodeFS.readFileSync(threadPath, "utf8"), '{"id":"accepted"}'); |
| 214 | } finally { |
| 215 | NodeFS.rmSync(tempDir, { recursive: true, force: true }); |
| 216 | } |
| 217 | }), |
| 218 | ); |
| 219 | |
| 220 | it.effect("summarizes large histories without reading their items", () => |
| 221 | Effect.gen(function* () { |
| 222 | const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-")); |
| 223 | const basePath = NodePath.join(tempDir, "events.log"); |
| 224 | const turns = Array.from({ length: 10_000 }); |
| 225 | Object.defineProperty(turns, 0, { |
| 226 | get: () => { |
| 227 | throw new Error("history must not be serialized"); |
| 228 | }, |
| 229 | }); |
| 230 | |
| 231 | try { |
| 232 | const store = yield* makeEventNdjsonLogStore(basePath); |
| 233 | yield* store.logger("native").write( |
| 234 | { |
| 235 | provider: "codex", |
| 236 | event: { |
| 237 | direction: "incoming", |
| 238 | stage: "decoded", |
| 239 | payload: { id: 42, result: { thread: { id: "native-thread", turns } } }, |
| 240 | }, |
| 241 | }, |
| 242 | ThreadId.make("large-history"), |
| 243 | ); |
| 244 | yield* store.close(); |
| 245 | |
| 246 | const contents = NodeFS.readFileSync(ownedLogPath(basePath, "large-history"), "utf8"); |
| 247 | assert.isBelow(Buffer.byteLength(contents), 2_048); |
| 248 | const record = decodeUnknownJson(parseLogLine(contents.trim()).payload); |
| 249 | assert.nestedPropertyVal(record, "event.payload.id", 42); |
| 250 | assert.nestedPropertyVal(record, "event.payload.result.thread.id", "native-thread"); |
| 251 | assert.nestedPropertyVal(record, "event.payload.result.thread.turns.itemCount", 10_000); |
| 252 | } finally { |
| 253 | NodeFS.rmSync(tempDir, { recursive: true, force: true }); |
| 254 | } |
| 255 | }), |
| 256 | ); |
| 257 | |
| 258 | it.effect("drops Cursor streaming deltas while keeping its lifecycle updates", () => |
| 259 | Effect.gen(function* () { |
| 260 | const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-")); |
| 261 | const basePath = NodePath.join(tempDir, "events.log"); |
| 262 | const threadId = ThreadId.make("cursor-deltas"); |
| 263 | const cursorUpdate = (update: object) => ({ |
| 264 | provider: "cursor", |
| 265 | kind: "protocol", |
| 266 | event: { |
| 267 | direction: "incoming", |
| 268 | stage: "decoded", |
| 269 | payload: { type: "interaction.update", runId: "run-1", update }, |
| 270 | }, |
| 271 | }); |
| 272 | const toolCall = { type: "shell", args: { command: "ls" } }; |
| 273 | const transient = [ |
| 274 | { type: "text-delta", text: "Hel" }, |
| 275 | { type: "thinking-delta", text: "Hmm" }, |
| 276 | { type: "token-delta", tokens: 3 }, |
| 277 | { type: "shell-output-delta", event: { stdout: "a.ts\n" } }, |
| 278 | { type: "partial-tool-call", callId: "call-1", modelCallId: "m-1", toolCall }, |
| 279 | { |
| 280 | type: "tool-call-delta", |
| 281 | callId: "call-1", |
| 282 | modelCallId: "m-1", |
| 283 | taskUpdate: { type: "text-delta", text: "Sub" }, |
| 284 | }, |
| 285 | ].map(cursorUpdate); |
| 286 | const retained = [ |
| 287 | { type: "tool-call-started", callId: "call-1", modelCallId: "m-1", toolCall }, |
| 288 | { |
| 289 | type: "tool-call-delta", |
| 290 | callId: "call-1", |
| 291 | modelCallId: "m-1", |
| 292 | taskUpdate: { type: "tool-call-completed", callId: "call-2", toolCall }, |
| 293 | }, |
| 294 | { type: "tool-call-completed", callId: "call-1", modelCallId: "m-1", toolCall }, |
| 295 | { type: "turn-ended" }, |
| 296 | ].map(cursorUpdate); |
| 297 | |
| 298 | try { |
| 299 | const store = yield* makeEventNdjsonLogStore(basePath, { batchWindowMs: 0 }); |
| 300 | const native = store.logger("native"); |
| 301 | for (const event of [...transient, ...retained]) { |
| 302 | yield* native.write(event, threadId); |
| 303 | } |
| 304 | yield* store.close(); |
| 305 | |
| 306 | const records = NodeFS.readFileSync(ownedLogPath(basePath, "cursor-deltas"), "utf8") |
| 307 | .trim() |
| 308 | .split("\n") |
| 309 | .map((line) => decodeUnknownJson(parseLogLine(line).payload)); |
| 310 | assert.deepEqual(records, retained); |
| 311 | } finally { |
| 312 | NodeFS.rmSync(tempDir, { recursive: true, force: true }); |
| 313 | } |
| 314 | }), |
| 315 | ); |
| 316 | |
| 317 | it.effect("bounds oversized records while retaining failure details", () => |
| 318 | Effect.gen(function* () { |
| 319 | const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-")); |
| 320 | const basePath = NodePath.join(tempDir, "events.log"); |
| 321 | |
| 322 | try { |
| 323 | const store = yield* makeEventNdjsonLogStore(basePath); |
| 324 | const logger = store.logger("native"); |
| 325 | const threadId = ThreadId.make("large-error"); |
| 326 | const failure = { |
| 327 | method: "error", |
| 328 | params: { |
| 329 | threadId: "native-thread", |
| 330 | turnId: "native-turn", |
| 331 | error: { message: "The provider is unavailable.", code: "overloaded" }, |
| 332 | output: "x".repeat(128 * 1_024), |
| 333 | }, |
| 334 | }; |
| 335 | yield* logger.write(failure, threadId); |
| 336 | yield* logger.write({ id: "escaped", output: "\u0000".repeat(20_000) }, threadId); |
| 337 | yield* store.close(); |
| 338 | |
| 339 | const contents = NodeFS.readFileSync(ownedLogPath(basePath, "large-error"), "utf8"); |
| 340 | const records = contents |
| 341 | .trim() |
| 342 | .split("\n") |
| 343 | .map((line) => decodeUnknownJson(parseLogLine(line).payload)); |
| 344 | assert.isBelow(Buffer.byteLength(contents), 64 * 1_024); |
| 345 | assert.equal(records.length, 2); |
| 346 | assert.nestedPropertyVal(records[0], "params.threadId", "native-thread"); |
| 347 | assert.nestedPropertyVal(records[0], "params.turnId", "native-turn"); |
| 348 | assert.nestedPropertyVal( |
| 349 | records[0], |
| 350 | "params.error.message", |
| 351 | "The provider is unavailable.", |
| 352 | ); |
| 353 | assert.nestedPropertyVal(records[0], "params.error.code", "overloaded"); |
| 354 | assert.propertyVal(records[1], "id", "escaped"); |
| 355 | } finally { |
| 356 | NodeFS.rmSync(tempDir, { recursive: true, force: true }); |
| 357 | } |
| 358 | }), |
| 359 | ); |
| 360 | |
| 361 | it.effect("bounds canonical diff snapshots before serializing their duplicate payloads", () => |
| 362 | Effect.gen(function* () { |
| 363 | const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-")); |
| 364 | const basePath = NodePath.join(tempDir, "events.log"); |
| 365 | const threadId = ThreadId.make("large-diff"); |
| 366 | const diff = "diff-payload".repeat(128 * 1_024); |
| 367 | try { |
| 368 | const store = yield* makeEventNdjsonLogStore(basePath, { batchWindowMs: 0 }); |
| 369 | yield* store.logger("canonical").write( |
| 370 | { |
| 371 | type: "turn.diff.updated", |
| 372 | threadId, |
| 373 | turnId: "native-turn", |
| 374 | raw: { method: "turn/diff/updated", payload: { diff } }, |
| 375 | payload: { unifiedDiff: diff }, |
| 376 | }, |
| 377 | threadId, |
| 378 | ); |
| 379 | yield* store.close(); |
| 380 | const contents = NodeFS.readFileSync(ownedLogPath(basePath, "large-diff"), "utf8"); |
| 381 | assert.isBelow(Buffer.byteLength(contents), 2_048); |
| 382 | const record = decodeUnknownJson(parseLogLine(contents.trim()).payload); |
| 383 | assert.propertyVal(record, "type", "turn.diff.updated"); |
| 384 | assert.propertyVal(record, "threadId", threadId); |
| 385 | assert.propertyVal(record, "turnId", "native-turn"); |
| 386 | } finally { |
| 387 | NodeFS.rmSync(tempDir, { recursive: true, force: true }); |
| 388 | } |
| 389 | }), |
| 390 | ); |
| 391 | |
| 392 | it.effect("serializes concurrent first writes for the same segment", () => |
| 393 | Effect.gen(function* () { |
| 394 | const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-")); |
| 395 | const basePath = NodePath.join(tempDir, "provider-canonical.ndjson"); |
| 396 | |
| 397 | try { |
| 398 | const store = yield* makeEventNdjsonLogStore(basePath, { batchWindowMs: 0 }); |
| 399 | const logger = store.logger("canonical"); |
| 400 | |
| 401 | yield* Effect.all( |
| 402 | [ |
| 403 | logger.write({ id: "evt-concurrent-1" }, null), |
| 404 | logger.write({ id: "evt-concurrent-2" }, null), |
| 405 | ], |
| 406 | { concurrency: "unbounded" }, |
| 407 | ); |
| 408 | yield* store.close(); |
| 409 | |
| 410 | const globalPath = ownedLogPath(basePath, "_global"); |
| 411 | assert.equal(NodeFS.existsSync(globalPath), true); |
| 412 | const lines = NodeFS.readFileSync(globalPath, "utf8") |
| 413 | .trim() |
| 414 | .split("\n") |
| 415 | .map((line) => parseLogLine(line)); |
| 416 | |
| 417 | assert.equal(lines.length, 2); |
| 418 | assert.deepEqual(lines.map((line) => line.payload).toSorted(), [ |
| 419 | '{"id":"evt-concurrent-1"}', |
| 420 | '{"id":"evt-concurrent-2"}', |
| 421 | ]); |
| 422 | } finally { |
| 423 | NodeFS.rmSync(tempDir, { recursive: true, force: true }); |
| 424 | } |
| 425 | }), |
| 426 | ); |
| 427 | |
| 428 | it.effect("rotates per-thread files when max size is exceeded", () => |
| 429 | Effect.gen(function* () { |
| 430 | const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-")); |
| 431 | const basePath = NodePath.join(tempDir, "provider-native.ndjson"); |
| 432 | |
| 433 | try { |
| 434 | const store = yield* makeEventNdjsonLogStore(basePath, { |
| 435 | maxBytes: 120, |
| 436 | maxFiles: 2, |
| 437 | batchWindowMs: 0, |
| 438 | }); |
| 439 | const native = store.logger("native"); |
| 440 | const canonical = store.logger("canonical"); |
| 441 | |
| 442 | for (let index = 0; index < 10; index += 1) { |
| 443 | const logger = index % 2 === 0 ? native : canonical; |
| 444 | yield* logger.write( |
| 445 | { |
| 446 | type: "session.started", |
| 447 | threadId: "provider-thread-rotate", |
| 448 | id: `evt-${index}`, |
| 449 | payload: "x".repeat(40), |
| 450 | }, |
| 451 | ThreadId.make("thread-rotate"), |
| 452 | ); |
| 453 | } |
| 454 | yield* store.close(); |
| 455 | |
| 456 | const fileStem = NodePath.basename(ownedLogPath(basePath, "thread-rotate")); |
| 457 | const matchingFiles = NodeFS.readdirSync(tempDir) |
| 458 | .filter((entry) => entry === fileStem || entry.startsWith(`${fileStem}.`)) |
| 459 | .toSorted(); |
| 460 | |
| 461 | assert.equal( |
| 462 | matchingFiles.some((entry) => entry === `${fileStem}.1`), |
| 463 | true, |
| 464 | ); |
| 465 | assert.equal( |
| 466 | matchingFiles.some((entry) => entry === fileStem || entry === `${fileStem}.2`), |
| 467 | true, |
| 468 | ); |
| 469 | assert.equal( |
| 470 | matchingFiles.some((entry) => entry === `${fileStem}.3`), |
| 471 | false, |
| 472 | ); |
| 473 | } finally { |
| 474 | NodeFS.rmSync(tempDir, { recursive: true, force: true }); |
| 475 | } |
| 476 | }), |
| 477 | ); |
| 478 | |
| 479 | it.effect("retires a thread log once a retention interval passes without writes", () => |
| 480 | Effect.gen(function* () { |
| 481 | const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-")); |
| 482 | const basePath = NodePath.join(tempDir, "events.log"); |
| 483 | const finishedPath = ownedLogPath(basePath, "finished"); |
| 484 | const activePath = ownedLogPath(basePath, "active"); |
| 485 | |
| 486 | try { |
| 487 | const store = yield* makeEventNdjsonLogStore(basePath, { |
| 488 | batchWindowMs: 0, |
| 489 | }); |
| 490 | const logger = store.logger("native"); |
| 491 | |
| 492 | yield* logger.write({ id: "finished-first" }, ThreadId.make("finished")); |
| 493 | yield* TestClock.adjust("5 minutes"); |
| 494 | yield* logger.write({ id: "active-first" }, ThreadId.make("active")); |
| 495 | assert.equal(NodeFS.existsSync(finishedPath), true); |
| 496 | NodeFS.truncateSync(finishedPath, 512 * 1024 * 1024 + 1); |
| 497 | |
| 498 | yield* TestClock.adjust("5 minutes"); |
| 499 | yield* logger.write({ id: "active-second" }, ThreadId.make("active")); |
| 500 | assert.equal(NodeFS.existsSync(finishedPath), false); |
| 501 | assert.equal(NodeFS.existsSync(activePath), true); |
| 502 | |
| 503 | yield* logger.write({ id: "finished-again" }, ThreadId.make("finished")); |
| 504 | yield* store.close(); |
| 505 | assert.deepEqual( |
| 506 | NodeFS.readFileSync(finishedPath, "utf8") |
| 507 | .trim() |
| 508 | .split("\n") |
| 509 | .map((line) => parseLogLine(line).payload), |
| 510 | ['{"id":"finished-again"}'], |
| 511 | ); |
| 512 | } finally { |
| 513 | NodeFS.rmSync(tempDir, { recursive: true, force: true }); |
| 514 | } |
| 515 | }), |
| 516 | ); |
| 517 | |
| 518 | it.effect("attributes batches that were written before a later chunk fails", () => |
| 519 | Effect.gen(function* () { |
| 520 | const records: ReadonlyArray<PendingRecord> = [ |
| 521 | { |
| 522 | stream: "native", |
| 523 | threadSegment: "thread", |
| 524 | line: "first", |
| 525 | bytes: 5, |
| 526 | }, |
| 527 | { |
| 528 | stream: "canonical", |
| 529 | threadSegment: "thread", |
| 530 | line: "second", |
| 531 | bytes: 6, |
| 532 | }, |
| 533 | ]; |
| 534 | const attributed: Array<PendingRecord> = []; |
| 535 | let writes = 0; |
| 536 | |
| 537 | const result = yield* writeBatchedMessages( |
| 538 | { |
| 539 | write: () => { |
| 540 | writes += 1; |
| 541 | return writes === 2 |
| 542 | ? Effect.fail( |
| 543 | new RotatingFileSinkError({ |
| 544 | operation: "write", |
| 545 | filePath: "test.log", |
| 546 | cause: "simulated disk exhaustion", |
| 547 | }), |
| 548 | ) |
| 549 | : Effect.void; |
| 550 | }, |
| 551 | }, |
| 552 | records, |
| 553 | 5, |
| 554 | (written) => attributed.push(...written), |
| 555 | ).pipe(Effect.result); |
| 556 | assert.equal(result._tag, "Failure"); |
| 557 | assert.deepEqual(attributed, [records[0]]); |
| 558 | }), |
| 559 | ); |
| 560 | |
| 561 | it.effect( |
| 562 | "keeps unrelated work responsive and drains before close while a disk write is held", |
| 563 | () => |
| 564 | Effect.gen(function* () { |
| 565 | const tempDir = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-provider-log-")); |
| 566 | const basePath = NodePath.join(tempDir, "events.log"); |
| 567 | const fs = yield* FileSystem.FileSystem; |
| 568 | const started = yield* Deferred.make<void>(); |
| 569 | const release = yield* Deferred.make<void>(); |
| 570 | let writes = 0; |
| 571 | try { |
| 572 | const store = yield* makeEventNdjsonLogStore(basePath, { batchWindowMs: 0 }).pipe( |
| 573 | Effect.provideService(FileSystem.FileSystem, { |
| 574 | ...fs, |
| 575 | writeFile: (filePath, content, options) => |
| 576 | Effect.gen(function* () { |
| 577 | if (++writes === 1) { |
| 578 | yield* Deferred.succeed(started, undefined); |
| 579 | yield* Deferred.await(release); |
| 580 | } |
| 581 | yield* fs.writeFile(filePath, content, options); |
| 582 | }), |
| 583 | }), |
| 584 | ); |
| 585 | const first = yield* store |
| 586 | .logger("native") |
| 587 | .write({ id: "first" }, ThreadId.make("held")) |
| 588 | .pipe(Effect.forkChild); |
| 589 | yield* Deferred.await(started); |
| 590 | const closed = yield* Deferred.make<void>(); |
| 591 | const close = yield* store |
| 592 | .close() |
| 593 | .pipe(Effect.andThen(Deferred.succeed(closed, undefined)), Effect.forkChild); |
| 594 | assert.equal(yield* Deferred.isDone(closed), false); |
| 595 | |
| 596 | const heartbeat = yield* Deferred.make<void>(); |
| 597 | yield* Effect.forkChild(Deferred.succeed(heartbeat, undefined)); |
| 598 | yield* Deferred.await(heartbeat); |
| 599 | yield* Deferred.succeed(release, undefined); |
| 600 | yield* Fiber.join(first); |
| 601 | yield* Fiber.join(close); |
| 602 | assert.include( |
| 603 | NodeFS.readFileSync(ownedLogPath(basePath, "held"), "utf8"), |
| 604 | '{"id":"first"}', |
| 605 | ); |
| 606 | } finally { |
| 607 | NodeFS.rmSync(tempDir, { recursive: true, force: true }); |
| 608 | } |
| 609 | }), |
| 610 | ); |
| 611 | }); |