The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 04:45 UTC
| 1 | import * as NodeStream from "@effect/platform-node-shared/NodeStream" |
| 2 | import { assert, describe, it } from "@effect/vitest" |
| 3 | import { Effect } from "effect" |
| 4 | import * as Array from "effect/Array" |
| 5 | import * as ByteSize from "effect/ByteSize" |
| 6 | import * as Channel from "effect/Channel" |
| 7 | import * as Console from "effect/Console" |
| 8 | import * as Stream from "effect/Stream" |
| 9 | import { Duplex, Readable, Transform } from "node:stream" |
| 10 | import * as Zlib from "node:zlib" |
| 11 | |
| 12 | describe("Stream", () => { |
| 13 | const destroyedOnRead = () => |
| 14 | new Readable({ |
| 15 | read() { |
| 16 | this.destroy() |
| 17 | } |
| 18 | }) |
| 19 | const errorCode = (error: unknown) => (error as NodeJS.ErrnoException).code |
| 20 | |
| 21 | for (const [chunkSize, expectedSize] of [[undefined, 4096], [2, 2], [Infinity, 10_000]] as const) { |
| 22 | it.effect(`fromReadable batches synchronous draining (${chunkSize ?? "default 4096"})`, () => |
| 23 | Effect.gen(function*() { |
| 24 | let reads = 0 |
| 25 | const readable = new Readable({ |
| 26 | objectMode: true, |
| 27 | highWaterMark: 1, |
| 28 | read() { |
| 29 | reads++ |
| 30 | this.push(reads > 10_000 ? null : reads) |
| 31 | } |
| 32 | }) |
| 33 | const chunks = yield* NodeStream.fromReadable<number>({ |
| 34 | evaluate: () => readable, |
| 35 | ...(chunkSize === undefined ? {} : { chunkSize }) |
| 36 | }).pipe(Stream.chunks, Stream.take(1), Stream.runCollect) |
| 37 | |
| 38 | assert.strictEqual(chunks.length, 1) |
| 39 | assert.strictEqual(chunks[0].length, expectedSize) |
| 40 | assert.deepEqual(chunks, [Array.range(1, expectedSize)]) |
| 41 | // Node may prefetch one object into its own buffer. |
| 42 | assert.isAtMost(reads, expectedSize + 1) |
| 43 | })) |
| 44 | } |
| 45 | |
| 46 | for ( |
| 47 | const [options, expected] of [ |
| 48 | [{ chunkSize: 2, readSize: 3 }, [["abc", "def"], ["ghi", "jk"]]], |
| 49 | [{ chunkSize: 2 }, [["abcdefghijk"]]], |
| 50 | [{ readSize: 3 }, [["abc", "def", "ghi", "jk"]]], |
| 51 | [{ chunkSize: Infinity, readSize: 3 }, [["abc", "def", "ghi", "jk"]]] |
| 52 | ] as const |
| 53 | ) { |
| 54 | it.effect(`fromReadable separates read size from batching (chunkSize=${"chunkSize" in options ? options.chunkSize : "default"}, readSize=${"readSize" in options ? options.readSize : "default"})`, () => |
| 55 | Effect.gen(function*() { |
| 56 | const readable = new Readable({ read() {} }) |
| 57 | readable.push(Buffer.from("abcdefghijk")) |
| 58 | readable.push(null) |
| 59 | const chunks = yield* NodeStream.fromReadable({ |
| 60 | evaluate: () => readable, |
| 61 | ...options |
| 62 | }).pipe(Stream.chunks, Stream.runCollect) |
| 63 | |
| 64 | assert.deepEqual( |
| 65 | chunks.map((batch) => batch.map((bytes) => Buffer.from(bytes).toString())), |
| 66 | expected.map((batch) => [...batch]) |
| 67 | ) |
| 68 | })) |
| 69 | } |
| 70 | |
| 71 | it.effect("fromReadable fails on premature close", () => |
| 72 | Effect.gen(function*() { |
| 73 | const code = yield* NodeStream.fromReadable({ evaluate: destroyedOnRead, onError: errorCode }).pipe( |
| 74 | Stream.runDrain, |
| 75 | Effect.flip |
| 76 | ) |
| 77 | assert.strictEqual(code, "ERR_STREAM_PREMATURE_CLOSE") |
| 78 | })) |
| 79 | |
| 80 | it.effect("fromReadable fails on an already destroyed readable", () => |
| 81 | Effect.gen(function*() { |
| 82 | const readable = new Readable({ read() {} }) |
| 83 | const closed = new Promise((resolve) => readable.once("close", resolve)) |
| 84 | readable.destroy() |
| 85 | yield* Effect.promise(() => closed) |
| 86 | const code = yield* NodeStream.fromReadable({ evaluate: () => readable, onError: errorCode }).pipe( |
| 87 | Stream.runDrain, |
| 88 | Effect.flip |
| 89 | ) |
| 90 | assert.strictEqual(code, "ERR_STREAM_PREMATURE_CLOSE") |
| 91 | })) |
| 92 | |
| 93 | it.effect("fromReadable removes its listeners when destroyed without a close event", () => |
| 94 | Effect.gen(function*() { |
| 95 | const readable = new Readable({ emitClose: false, read() {} }) |
| 96 | readable.push("a") |
| 97 | yield* NodeStream.fromReadable({ evaluate: () => readable }).pipe(Stream.take(1), Stream.runDrain) |
| 98 | yield* Effect.promise(() => new Promise((resolve) => setImmediate(resolve))) |
| 99 | assert.strictEqual(readable.listenerCount("error"), 0) |
| 100 | assert.strictEqual(readable.listenerCount("close"), 0) |
| 101 | })) |
| 102 | |
| 103 | it.effect("toString fails on premature close", () => |
| 104 | Effect.gen(function*() { |
| 105 | const code = yield* NodeStream.toString(destroyedOnRead, { onError: errorCode }).pipe(Effect.flip) |
| 106 | assert.strictEqual(code, "ERR_STREAM_PREMATURE_CLOSE") |
| 107 | })) |
| 108 | |
| 109 | it.effect("toArrayBuffer fails on premature close", () => |
| 110 | Effect.gen(function*() { |
| 111 | const code = yield* NodeStream.toArrayBuffer(destroyedOnRead, { onError: errorCode }).pipe(Effect.flip) |
| 112 | assert.strictEqual(code, "ERR_STREAM_PREMATURE_CLOSE") |
| 113 | })) |
| 114 | |
| 115 | it.effect("toString keeps the maxBytes failure when destroy fails", () => |
| 116 | Effect.gen(function*() { |
| 117 | const readable = new Readable({ |
| 118 | read() { |
| 119 | this.push("too large") |
| 120 | }, |
| 121 | destroy(_error, callback) { |
| 122 | setImmediate(() => callback(new Error("destroy failed"))) |
| 123 | } |
| 124 | }) |
| 125 | const closed = new Promise((resolve) => readable.once("close", resolve)) |
| 126 | const error = yield* NodeStream.toString(() => readable, { |
| 127 | maxBytes: 1, |
| 128 | onError: (error) => (error as Error).message |
| 129 | }).pipe(Effect.flip) |
| 130 | assert.strictEqual(error, "maxBytes exceeded") |
| 131 | yield* Effect.promise(() => closed) |
| 132 | })) |
| 133 | |
| 134 | it.effect("handles a late upload error after overflow without leaking listeners", () => |
| 135 | Effect.gen(function*() { |
| 136 | const readable = new Readable({ read() {} }) |
| 137 | yield* Effect.addFinalizer(() => Effect.sync(() => readable.destroy())) |
| 138 | readable.push(Buffer.from("too large")) |
| 139 | const error = yield* NodeStream.toString(() => readable, { |
| 140 | maxBytes: 1, |
| 141 | destroyOnOverflow: false, |
| 142 | onError: (error) => error |
| 143 | }).pipe(Effect.flip) |
| 144 | |
| 145 | assert.instanceOf(error, NodeStream.MaxBytesError) |
| 146 | assert.isFalse(readable.destroyed) |
| 147 | readable.push(Buffer.from("remaining upload")) |
| 148 | const closed = new Promise<void>((resolve) => readable.once("close", resolve)) |
| 149 | readable.destroy(new Error("late upload failure")) |
| 150 | yield* Effect.promise(() => closed) |
| 151 | for (const event of ["data", "end", "error", "close", "finish"]) { |
| 152 | assert.strictEqual(readable.listenerCount(event), 0) |
| 153 | } |
| 154 | })) |
| 155 | |
| 156 | it.effect("should read a stream", () => |
| 157 | Effect.gen(function*() { |
| 158 | const stream = NodeStream.fromReadable<"error", string>({ |
| 159 | evaluate: () => Readable.from(["a", "b", "c"]), |
| 160 | onError: () => "error" |
| 161 | }) |
| 162 | const items = yield* Stream.runCollect(stream) |
| 163 | assert.deepEqual(items, ["a", "b", "c"]) |
| 164 | })) |
| 165 | |
| 166 | it.effect("fromDuplex", () => |
| 167 | Effect.gen(function*() { |
| 168 | const result = yield* Stream.fromArray(["a", "b", "c"]).pipe( |
| 169 | Stream.pipeThroughChannelOrFail(NodeStream.fromDuplex({ |
| 170 | evaluate: () => |
| 171 | new Transform({ |
| 172 | transform(chunk, _encoding, callback) { |
| 173 | callback(null, chunk.toString().toUpperCase()) |
| 174 | } |
| 175 | }), |
| 176 | onError: () => "error" as const |
| 177 | })), |
| 178 | Stream.decodeText(), |
| 179 | Stream.mkString |
| 180 | ) |
| 181 | |
| 182 | assert.strictEqual(result, "ABC") |
| 183 | })) |
| 184 | |
| 185 | for (const adapter of ["fromDuplex", "pipeThroughDuplex"] as const) { |
| 186 | it.effect(`${adapter} bounds byte drains and completes in order`, () => |
| 187 | Effect.gen(function*() { |
| 188 | const input = Buffer.from("abcdefghijk") |
| 189 | let offset = 0 |
| 190 | let reads = 0 |
| 191 | const options = { |
| 192 | evaluate: () => |
| 193 | new Duplex({ |
| 194 | highWaterMark: 1, |
| 195 | read() { |
| 196 | reads++ |
| 197 | if (offset === input.length) { |
| 198 | this.push(null) |
| 199 | } else { |
| 200 | const next = Math.min(offset + 3, input.length) |
| 201 | this.push(input.subarray(offset, next)) |
| 202 | offset = next |
| 203 | } |
| 204 | }, |
| 205 | write(_chunk, _encoding, callback) { |
| 206 | callback() |
| 207 | } |
| 208 | }), |
| 209 | chunkSize: 2, |
| 210 | readSize: 3 |
| 211 | } |
| 212 | const stream = adapter === "fromDuplex" |
| 213 | ? Stream.empty.pipe(Stream.pipeThroughChannelOrFail(NodeStream.fromDuplex(options))) |
| 214 | : Stream.empty.pipe(NodeStream.pipeThroughDuplex(options)) |
| 215 | let batches = 0 |
| 216 | const chunks = yield* stream.pipe( |
| 217 | Stream.chunks, |
| 218 | Stream.tap((batch) => |
| 219 | Effect.sync(() => { |
| 220 | assert.isAtMost(batch.length, 2) |
| 221 | if (batches++ === 0) { |
| 222 | assert.deepEqual(batch.map((bytes) => Buffer.from(bytes).toString()), ["abc", "def"]) |
| 223 | // Allow one Node-prefetched buffer, but not the entire source. |
| 224 | assert.isAtMost(reads, 3) |
| 225 | } |
| 226 | }) |
| 227 | ), |
| 228 | Stream.runCollect |
| 229 | ) |
| 230 | assert.isAbove(batches, 1) |
| 231 | const bytes = chunks.flat() |
| 232 | assert.deepEqual(bytes.map((chunk) => chunk.length), [3, 3, 3, 2]) |
| 233 | assert.strictEqual(Buffer.concat(bytes).toString(), "abcdefghijk") |
| 234 | })) |
| 235 | } |
| 236 | |
| 237 | it.effect("fromDuplex failure", () => |
| 238 | Effect.gen(function*() { |
| 239 | const result = yield* Stream.fromArray(["a", "b", "c"]).pipe( |
| 240 | Stream.pipeThroughChannelOrFail(NodeStream.fromDuplex({ |
| 241 | evaluate: () => |
| 242 | new Transform({ |
| 243 | transform(_chunk, _encoding, callback) { |
| 244 | callback(new Error()) |
| 245 | } |
| 246 | }), |
| 247 | onError: () => "error" as const |
| 248 | })), |
| 249 | Stream.runDrain, |
| 250 | Effect.flip |
| 251 | ) |
| 252 | |
| 253 | assert.strictEqual(result, "error") |
| 254 | })) |
| 255 | |
| 256 | it.effect("pipeThroughDuplex", () => |
| 257 | Effect.gen(function*() { |
| 258 | const result = yield* Stream.fromArray(["a", "b", "c"]).pipe( |
| 259 | NodeStream.pipeThroughDuplex({ |
| 260 | evaluate: () => |
| 261 | new Transform({ |
| 262 | transform(chunk, _encoding, callback) { |
| 263 | callback(null, chunk.toString().toUpperCase()) |
| 264 | } |
| 265 | }), |
| 266 | onError: () => "error" as const |
| 267 | }), |
| 268 | Stream.decodeText(), |
| 269 | Stream.mkString |
| 270 | ) |
| 271 | |
| 272 | assert.strictEqual(result, "ABC") |
| 273 | })) |
| 274 | |
| 275 | it.effect("pipeThroughDuplex propagates upstream failure", () => |
| 276 | Effect.gen(function*() { |
| 277 | const result = yield* Stream.fail("upstream error").pipe( |
| 278 | NodeStream.pipeThroughDuplex({ |
| 279 | evaluate: () => |
| 280 | new Duplex({ |
| 281 | read() {}, |
| 282 | write(_chunk, _encoding, callback) { |
| 283 | callback() |
| 284 | } |
| 285 | }) |
| 286 | }), |
| 287 | Stream.runDrain, |
| 288 | Effect.flip |
| 289 | ) |
| 290 | |
| 291 | assert.strictEqual(result, "upstream error") |
| 292 | })) |
| 293 | |
| 294 | it.effect("pipeThroughDuplex write error", () => |
| 295 | Effect.gen(function*() { |
| 296 | const result = yield* Stream.fromArray(["a", "b", "c"]).pipe( |
| 297 | NodeStream.pipeThroughDuplex({ |
| 298 | evaluate: () => |
| 299 | new Duplex({ |
| 300 | read() {}, |
| 301 | write(_chunk, _encoding, callback) { |
| 302 | callback(new Error()) |
| 303 | } |
| 304 | }), |
| 305 | onError: () => "error" as const |
| 306 | }), |
| 307 | Stream.runDrain, |
| 308 | Effect.flip |
| 309 | ) |
| 310 | assert.strictEqual(result, "error") |
| 311 | })) |
| 312 | |
| 313 | it.effect("pipeThroughSimple", () => |
| 314 | Effect.gen(function*() { |
| 315 | const result = yield* Stream.fromArray(["a", Buffer.from("b"), "c"]).pipe( |
| 316 | NodeStream.pipeThroughSimple( |
| 317 | () => |
| 318 | new Transform({ |
| 319 | transform(chunk, _encoding, callback) { |
| 320 | callback(null, chunk.toString().toUpperCase()) |
| 321 | } |
| 322 | }) |
| 323 | ), |
| 324 | Stream.decodeText(), |
| 325 | Stream.mkString |
| 326 | ) |
| 327 | |
| 328 | assert.strictEqual(result, "ABC") |
| 329 | })) |
| 330 | |
| 331 | it.effect("fromDuplex should work with node:zlib", () => |
| 332 | Effect.gen(function*() { |
| 333 | const text = "abcdefg1234567890" |
| 334 | const encoder = new TextEncoder() |
| 335 | const input = encoder.encode(text) |
| 336 | const stream = NodeStream.fromReadable<Uint8Array, "error">({ |
| 337 | evaluate: () => Readable.from([input]), |
| 338 | onError: () => "error" |
| 339 | }) |
| 340 | const deflate = NodeStream.fromDuplex({ |
| 341 | evaluate: () => Zlib.createGzip(), |
| 342 | onError: () => "error" as const |
| 343 | }) |
| 344 | const inflate = NodeStream.fromDuplex<never, Uint8Array, Uint8Array, "error">({ |
| 345 | evaluate: () => Zlib.createUnzip(), |
| 346 | onError: () => "error" as const |
| 347 | }) |
| 348 | const channel = Channel.pipeToOrFail(deflate, inflate) |
| 349 | const result = yield* stream.pipe( |
| 350 | Stream.pipeThroughChannelOrFail(channel), |
| 351 | Stream.decodeText(), |
| 352 | Stream.mkString |
| 353 | ) |
| 354 | assert.strictEqual(result, text) |
| 355 | })) |
| 356 | |
| 357 | it.effect("toReadable roundtrip", () => |
| 358 | Effect.gen(function*() { |
| 359 | const stream = Stream.range(0, 10000).pipe( |
| 360 | Stream.map((n) => String(n)) |
| 361 | ) |
| 362 | const readable = yield* NodeStream.toReadable(stream) |
| 363 | const outStream = NodeStream.fromReadable({ |
| 364 | evaluate: () => readable, |
| 365 | onError: () => "error" as const |
| 366 | }) |
| 367 | const items = yield* outStream.pipe( |
| 368 | Stream.decodeText(), |
| 369 | Stream.runCollect |
| 370 | ) |
| 371 | assert.strictEqual(items.join(""), Array.range(0, 10000).join("")) |
| 372 | })) |
| 373 | |
| 374 | it.effect("toReadable with error", () => |
| 375 | Effect.gen(function*() { |
| 376 | const stream = Stream.fail("error") |
| 377 | const readable = yield* NodeStream.toReadable(stream) |
| 378 | const outStream = NodeStream.fromReadable({ |
| 379 | evaluate: () => readable |
| 380 | }) |
| 381 | const error = yield* outStream.pipe( |
| 382 | Stream.runCollect, |
| 383 | Effect.tapError((_) => Console.log(_)), |
| 384 | Effect.flip |
| 385 | ) |
| 386 | assert.deepEqual(error.cause, "error") |
| 387 | })) |
| 388 | |
| 389 | it.effect("toString registers one error listener", () => |
| 390 | Effect.gen(function*() { |
| 391 | const stream = new Readable({ |
| 392 | read() {} |
| 393 | }) |
| 394 | yield* NodeStream.toString(() => stream).pipe(Effect.forkChild) |
| 395 | yield* Effect.yieldNow |
| 396 | assert.strictEqual(stream.listenerCount("error"), 1) |
| 397 | })) |
| 398 | |
| 399 | it.effect("collects strings and array buffers with an Infinity maxBytes limit", () => |
| 400 | Effect.gen(function*() { |
| 401 | const chunks = [Buffer.from("hello "), Buffer.from("world")] |
| 402 | const text = yield* NodeStream.toString(() => Readable.from(chunks), { maxBytes: Infinity }) |
| 403 | const buffer = yield* NodeStream.toArrayBuffer(() => Readable.from(chunks), { maxBytes: Infinity }) |
| 404 | |
| 405 | assert.strictEqual(text, "hello world") |
| 406 | assert.deepStrictEqual(new Uint8Array(buffer), new TextEncoder().encode("hello world")) |
| 407 | })) |
| 408 | |
| 409 | it.effect("toString enforces a zero maxBytes limit", () => |
| 410 | Effect.gen(function*() { |
| 411 | let emitted = false |
| 412 | const stream = new Readable({ |
| 413 | read() { |
| 414 | if (emitted) return |
| 415 | emitted = true |
| 416 | this.push("a") |
| 417 | } |
| 418 | }) |
| 419 | const error = yield* NodeStream.toString(() => stream, { |
| 420 | maxBytes: "0 B", |
| 421 | onError: () => "maxBytes exceeded" as const |
| 422 | }).pipe(Effect.flip) |
| 423 | |
| 424 | assert.strictEqual(error, "maxBytes exceeded") |
| 425 | assert.isTrue(stream.destroyed) |
| 426 | })) |
| 427 | |
| 428 | it.effect("toString limits the bytes read before decoding", () => |
| 429 | Effect.gen(function*() { |
| 430 | const latin1 = yield* NodeStream.toString(() => Readable.from([Buffer.from([0xe9])]), { |
| 431 | encoding: "latin1", |
| 432 | maxBytes: 1 |
| 433 | }) |
| 434 | const utf16le = yield* NodeStream.toString(() => Readable.from([Buffer.from("abcd", "utf16le")]), { |
| 435 | encoding: "utf16le", |
| 436 | maxBytes: 7, |
| 437 | onError: (error) => error |
| 438 | }).pipe(Effect.flip) |
| 439 | |
| 440 | assert.strictEqual(latin1, "é") |
| 441 | assert.instanceOf(utf16le, NodeStream.MaxBytesError) |
| 442 | })) |
| 443 | |
| 444 | it.effect("toString decodes characters split across chunks", () => |
| 445 | Effect.gen(function*() { |
| 446 | const chunks = [Buffer.from([0xe2, 0x82]), Buffer.from([0xac, 0xe2])] |
| 447 | const text = yield* NodeStream.toString(() => Readable.from(chunks), { maxBytes: 4 }) |
| 448 | |
| 449 | assert.strictEqual(text, "€\ufffd") |
| 450 | })) |
| 451 | |
| 452 | it.effect("toString limits the bytes read by a stream that already decodes", () => |
| 453 | Effect.gen(function*() { |
| 454 | const stream = new Readable({ |
| 455 | read() { |
| 456 | this.push(Buffer.from([0xe9])) |
| 457 | this.push(null) |
| 458 | } |
| 459 | }) |
| 460 | stream.setEncoding("utf8") |
| 461 | const text = yield* NodeStream.toString(() => stream, { encoding: "latin1", maxBytes: 1 }) |
| 462 | |
| 463 | assert.strictEqual(text, "é") |
| 464 | })) |
| 465 | |
| 466 | it.effect("toString limits object-mode strings by their UTF-8 length", () => |
| 467 | Effect.gen(function*() { |
| 468 | const error = yield* NodeStream.toString(() => Readable.from(["a", "b", "c", "d"]), { |
| 469 | encoding: "hex", |
| 470 | maxBytes: 3, |
| 471 | onError: (error) => error |
| 472 | }).pipe(Effect.flip) |
| 473 | |
| 474 | assert.instanceOf(error, NodeStream.MaxBytesError) |
| 475 | })) |
| 476 | |
| 477 | it.effect("toArrayBuffer enforces a zero maxBytes limit", () => |
| 478 | Effect.gen(function*() { |
| 479 | let emitted = false |
| 480 | const stream = new Readable({ |
| 481 | read() { |
| 482 | if (emitted) return |
| 483 | emitted = true |
| 484 | this.push(Buffer.from("a")) |
| 485 | } |
| 486 | }) |
| 487 | const error = yield* NodeStream.toArrayBuffer(() => stream, { |
| 488 | maxBytes: ByteSize.zero, |
| 489 | onError: () => "maxBytes exceeded" as const |
| 490 | }).pipe(Effect.flip) |
| 491 | |
| 492 | assert.strictEqual(error, "maxBytes exceeded") |
| 493 | assert.isTrue(stream.destroyed) |
| 494 | })) |
| 495 | }) |