@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 495 lines 16.9 KB
Raw
1 import * as NodeStream from "@effect/platform-node-shared/NodeStream"
2 import { assert, describe, it } from "@effect/vitest"
3 import { Effect } from "effect"
4 import * as Array from "effect/Array"
5 import * as ByteSize from "effect/ByteSize"
6 import * as Channel from "effect/Channel"
7 import * as Console from "effect/Console"
8 import * as Stream from "effect/Stream"
9 import { Duplex, Readable, Transform } from "node:stream"
10 import * as Zlib from "node:zlib"
11
12 describe("Stream", () => {
13 const destroyedOnRead = () =>
14 new Readable({
15 read() {
16 this.destroy()
17 }
18 })
19 const errorCode = (error: unknown) => (error as NodeJS.ErrnoException).code
20
21 for (const [chunkSize, expectedSize] of [[undefined, 4096], [2, 2], [Infinity, 10_000]] as const) {
22 it.effect(`fromReadable batches synchronous draining (${chunkSize ?? "default 4096"})`, () =>
23 Effect.gen(function*() {
24 let reads = 0
25 const readable = new Readable({
26 objectMode: true,
27 highWaterMark: 1,
28 read() {
29 reads++
30 this.push(reads > 10_000 ? null : reads)
31 }
32 })
33 const chunks = yield* NodeStream.fromReadable<number>({
34 evaluate: () => readable,
35 ...(chunkSize === undefined ? {} : { chunkSize })
36 }).pipe(Stream.chunks, Stream.take(1), Stream.runCollect)
37
38 assert.strictEqual(chunks.length, 1)
39 assert.strictEqual(chunks[0].length, expectedSize)
40 assert.deepEqual(chunks, [Array.range(1, expectedSize)])
41 // Node may prefetch one object into its own buffer.
42 assert.isAtMost(reads, expectedSize + 1)
43 }))
44 }
45
46 for (
47 const [options, expected] of [
48 [{ chunkSize: 2, readSize: 3 }, [["abc", "def"], ["ghi", "jk"]]],
49 [{ chunkSize: 2 }, [["abcdefghijk"]]],
50 [{ readSize: 3 }, [["abc", "def", "ghi", "jk"]]],
51 [{ chunkSize: Infinity, readSize: 3 }, [["abc", "def", "ghi", "jk"]]]
52 ] as const
53 ) {
54 it.effect(`fromReadable separates read size from batching (chunkSize=${"chunkSize" in options ? options.chunkSize : "default"}, readSize=${"readSize" in options ? options.readSize : "default"})`, () =>
55 Effect.gen(function*() {
56 const readable = new Readable({ read() {} })
57 readable.push(Buffer.from("abcdefghijk"))
58 readable.push(null)
59 const chunks = yield* NodeStream.fromReadable({
60 evaluate: () => readable,
61 ...options
62 }).pipe(Stream.chunks, Stream.runCollect)
63
64 assert.deepEqual(
65 chunks.map((batch) => batch.map((bytes) => Buffer.from(bytes).toString())),
66 expected.map((batch) => [...batch])
67 )
68 }))
69 }
70
71 it.effect("fromReadable fails on premature close", () =>
72 Effect.gen(function*() {
73 const code = yield* NodeStream.fromReadable({ evaluate: destroyedOnRead, onError: errorCode }).pipe(
74 Stream.runDrain,
75 Effect.flip
76 )
77 assert.strictEqual(code, "ERR_STREAM_PREMATURE_CLOSE")
78 }))
79
80 it.effect("fromReadable fails on an already destroyed readable", () =>
81 Effect.gen(function*() {
82 const readable = new Readable({ read() {} })
83 const closed = new Promise((resolve) => readable.once("close", resolve))
84 readable.destroy()
85 yield* Effect.promise(() => closed)
86 const code = yield* NodeStream.fromReadable({ evaluate: () => readable, onError: errorCode }).pipe(
87 Stream.runDrain,
88 Effect.flip
89 )
90 assert.strictEqual(code, "ERR_STREAM_PREMATURE_CLOSE")
91 }))
92
93 it.effect("fromReadable removes its listeners when destroyed without a close event", () =>
94 Effect.gen(function*() {
95 const readable = new Readable({ emitClose: false, read() {} })
96 readable.push("a")
97 yield* NodeStream.fromReadable({ evaluate: () => readable }).pipe(Stream.take(1), Stream.runDrain)
98 yield* Effect.promise(() => new Promise((resolve) => setImmediate(resolve)))
99 assert.strictEqual(readable.listenerCount("error"), 0)
100 assert.strictEqual(readable.listenerCount("close"), 0)
101 }))
102
103 it.effect("toString fails on premature close", () =>
104 Effect.gen(function*() {
105 const code = yield* NodeStream.toString(destroyedOnRead, { onError: errorCode }).pipe(Effect.flip)
106 assert.strictEqual(code, "ERR_STREAM_PREMATURE_CLOSE")
107 }))
108
109 it.effect("toArrayBuffer fails on premature close", () =>
110 Effect.gen(function*() {
111 const code = yield* NodeStream.toArrayBuffer(destroyedOnRead, { onError: errorCode }).pipe(Effect.flip)
112 assert.strictEqual(code, "ERR_STREAM_PREMATURE_CLOSE")
113 }))
114
115 it.effect("toString keeps the maxBytes failure when destroy fails", () =>
116 Effect.gen(function*() {
117 const readable = new Readable({
118 read() {
119 this.push("too large")
120 },
121 destroy(_error, callback) {
122 setImmediate(() => callback(new Error("destroy failed")))
123 }
124 })
125 const closed = new Promise((resolve) => readable.once("close", resolve))
126 const error = yield* NodeStream.toString(() => readable, {
127 maxBytes: 1,
128 onError: (error) => (error as Error).message
129 }).pipe(Effect.flip)
130 assert.strictEqual(error, "maxBytes exceeded")
131 yield* Effect.promise(() => closed)
132 }))
133
134 it.effect("handles a late upload error after overflow without leaking listeners", () =>
135 Effect.gen(function*() {
136 const readable = new Readable({ read() {} })
137 yield* Effect.addFinalizer(() => Effect.sync(() => readable.destroy()))
138 readable.push(Buffer.from("too large"))
139 const error = yield* NodeStream.toString(() => readable, {
140 maxBytes: 1,
141 destroyOnOverflow: false,
142 onError: (error) => error
143 }).pipe(Effect.flip)
144
145 assert.instanceOf(error, NodeStream.MaxBytesError)
146 assert.isFalse(readable.destroyed)
147 readable.push(Buffer.from("remaining upload"))
148 const closed = new Promise<void>((resolve) => readable.once("close", resolve))
149 readable.destroy(new Error("late upload failure"))
150 yield* Effect.promise(() => closed)
151 for (const event of ["data", "end", "error", "close", "finish"]) {
152 assert.strictEqual(readable.listenerCount(event), 0)
153 }
154 }))
155
156 it.effect("should read a stream", () =>
157 Effect.gen(function*() {
158 const stream = NodeStream.fromReadable<"error", string>({
159 evaluate: () => Readable.from(["a", "b", "c"]),
160 onError: () => "error"
161 })
162 const items = yield* Stream.runCollect(stream)
163 assert.deepEqual(items, ["a", "b", "c"])
164 }))
165
166 it.effect("fromDuplex", () =>
167 Effect.gen(function*() {
168 const result = yield* Stream.fromArray(["a", "b", "c"]).pipe(
169 Stream.pipeThroughChannelOrFail(NodeStream.fromDuplex({
170 evaluate: () =>
171 new Transform({
172 transform(chunk, _encoding, callback) {
173 callback(null, chunk.toString().toUpperCase())
174 }
175 }),
176 onError: () => "error" as const
177 })),
178 Stream.decodeText(),
179 Stream.mkString
180 )
181
182 assert.strictEqual(result, "ABC")
183 }))
184
185 for (const adapter of ["fromDuplex", "pipeThroughDuplex"] as const) {
186 it.effect(`${adapter} bounds byte drains and completes in order`, () =>
187 Effect.gen(function*() {
188 const input = Buffer.from("abcdefghijk")
189 let offset = 0
190 let reads = 0
191 const options = {
192 evaluate: () =>
193 new Duplex({
194 highWaterMark: 1,
195 read() {
196 reads++
197 if (offset === input.length) {
198 this.push(null)
199 } else {
200 const next = Math.min(offset + 3, input.length)
201 this.push(input.subarray(offset, next))
202 offset = next
203 }
204 },
205 write(_chunk, _encoding, callback) {
206 callback()
207 }
208 }),
209 chunkSize: 2,
210 readSize: 3
211 }
212 const stream = adapter === "fromDuplex"
213 ? Stream.empty.pipe(Stream.pipeThroughChannelOrFail(NodeStream.fromDuplex(options)))
214 : Stream.empty.pipe(NodeStream.pipeThroughDuplex(options))
215 let batches = 0
216 const chunks = yield* stream.pipe(
217 Stream.chunks,
218 Stream.tap((batch) =>
219 Effect.sync(() => {
220 assert.isAtMost(batch.length, 2)
221 if (batches++ === 0) {
222 assert.deepEqual(batch.map((bytes) => Buffer.from(bytes).toString()), ["abc", "def"])
223 // Allow one Node-prefetched buffer, but not the entire source.
224 assert.isAtMost(reads, 3)
225 }
226 })
227 ),
228 Stream.runCollect
229 )
230 assert.isAbove(batches, 1)
231 const bytes = chunks.flat()
232 assert.deepEqual(bytes.map((chunk) => chunk.length), [3, 3, 3, 2])
233 assert.strictEqual(Buffer.concat(bytes).toString(), "abcdefghijk")
234 }))
235 }
236
237 it.effect("fromDuplex failure", () =>
238 Effect.gen(function*() {
239 const result = yield* Stream.fromArray(["a", "b", "c"]).pipe(
240 Stream.pipeThroughChannelOrFail(NodeStream.fromDuplex({
241 evaluate: () =>
242 new Transform({
243 transform(_chunk, _encoding, callback) {
244 callback(new Error())
245 }
246 }),
247 onError: () => "error" as const
248 })),
249 Stream.runDrain,
250 Effect.flip
251 )
252
253 assert.strictEqual(result, "error")
254 }))
255
256 it.effect("pipeThroughDuplex", () =>
257 Effect.gen(function*() {
258 const result = yield* Stream.fromArray(["a", "b", "c"]).pipe(
259 NodeStream.pipeThroughDuplex({
260 evaluate: () =>
261 new Transform({
262 transform(chunk, _encoding, callback) {
263 callback(null, chunk.toString().toUpperCase())
264 }
265 }),
266 onError: () => "error" as const
267 }),
268 Stream.decodeText(),
269 Stream.mkString
270 )
271
272 assert.strictEqual(result, "ABC")
273 }))
274
275 it.effect("pipeThroughDuplex propagates upstream failure", () =>
276 Effect.gen(function*() {
277 const result = yield* Stream.fail("upstream error").pipe(
278 NodeStream.pipeThroughDuplex({
279 evaluate: () =>
280 new Duplex({
281 read() {},
282 write(_chunk, _encoding, callback) {
283 callback()
284 }
285 })
286 }),
287 Stream.runDrain,
288 Effect.flip
289 )
290
291 assert.strictEqual(result, "upstream error")
292 }))
293
294 it.effect("pipeThroughDuplex write error", () =>
295 Effect.gen(function*() {
296 const result = yield* Stream.fromArray(["a", "b", "c"]).pipe(
297 NodeStream.pipeThroughDuplex({
298 evaluate: () =>
299 new Duplex({
300 read() {},
301 write(_chunk, _encoding, callback) {
302 callback(new Error())
303 }
304 }),
305 onError: () => "error" as const
306 }),
307 Stream.runDrain,
308 Effect.flip
309 )
310 assert.strictEqual(result, "error")
311 }))
312
313 it.effect("pipeThroughSimple", () =>
314 Effect.gen(function*() {
315 const result = yield* Stream.fromArray(["a", Buffer.from("b"), "c"]).pipe(
316 NodeStream.pipeThroughSimple(
317 () =>
318 new Transform({
319 transform(chunk, _encoding, callback) {
320 callback(null, chunk.toString().toUpperCase())
321 }
322 })
323 ),
324 Stream.decodeText(),
325 Stream.mkString
326 )
327
328 assert.strictEqual(result, "ABC")
329 }))
330
331 it.effect("fromDuplex should work with node:zlib", () =>
332 Effect.gen(function*() {
333 const text = "abcdefg1234567890"
334 const encoder = new TextEncoder()
335 const input = encoder.encode(text)
336 const stream = NodeStream.fromReadable<Uint8Array, "error">({
337 evaluate: () => Readable.from([input]),
338 onError: () => "error"
339 })
340 const deflate = NodeStream.fromDuplex({
341 evaluate: () => Zlib.createGzip(),
342 onError: () => "error" as const
343 })
344 const inflate = NodeStream.fromDuplex<never, Uint8Array, Uint8Array, "error">({
345 evaluate: () => Zlib.createUnzip(),
346 onError: () => "error" as const
347 })
348 const channel = Channel.pipeToOrFail(deflate, inflate)
349 const result = yield* stream.pipe(
350 Stream.pipeThroughChannelOrFail(channel),
351 Stream.decodeText(),
352 Stream.mkString
353 )
354 assert.strictEqual(result, text)
355 }))
356
357 it.effect("toReadable roundtrip", () =>
358 Effect.gen(function*() {
359 const stream = Stream.range(0, 10000).pipe(
360 Stream.map((n) => String(n))
361 )
362 const readable = yield* NodeStream.toReadable(stream)
363 const outStream = NodeStream.fromReadable({
364 evaluate: () => readable,
365 onError: () => "error" as const
366 })
367 const items = yield* outStream.pipe(
368 Stream.decodeText(),
369 Stream.runCollect
370 )
371 assert.strictEqual(items.join(""), Array.range(0, 10000).join(""))
372 }))
373
374 it.effect("toReadable with error", () =>
375 Effect.gen(function*() {
376 const stream = Stream.fail("error")
377 const readable = yield* NodeStream.toReadable(stream)
378 const outStream = NodeStream.fromReadable({
379 evaluate: () => readable
380 })
381 const error = yield* outStream.pipe(
382 Stream.runCollect,
383 Effect.tapError((_) => Console.log(_)),
384 Effect.flip
385 )
386 assert.deepEqual(error.cause, "error")
387 }))
388
389 it.effect("toString registers one error listener", () =>
390 Effect.gen(function*() {
391 const stream = new Readable({
392 read() {}
393 })
394 yield* NodeStream.toString(() => stream).pipe(Effect.forkChild)
395 yield* Effect.yieldNow
396 assert.strictEqual(stream.listenerCount("error"), 1)
397 }))
398
399 it.effect("collects strings and array buffers with an Infinity maxBytes limit", () =>
400 Effect.gen(function*() {
401 const chunks = [Buffer.from("hello "), Buffer.from("world")]
402 const text = yield* NodeStream.toString(() => Readable.from(chunks), { maxBytes: Infinity })
403 const buffer = yield* NodeStream.toArrayBuffer(() => Readable.from(chunks), { maxBytes: Infinity })
404
405 assert.strictEqual(text, "hello world")
406 assert.deepStrictEqual(new Uint8Array(buffer), new TextEncoder().encode("hello world"))
407 }))
408
409 it.effect("toString enforces a zero maxBytes limit", () =>
410 Effect.gen(function*() {
411 let emitted = false
412 const stream = new Readable({
413 read() {
414 if (emitted) return
415 emitted = true
416 this.push("a")
417 }
418 })
419 const error = yield* NodeStream.toString(() => stream, {
420 maxBytes: "0 B",
421 onError: () => "maxBytes exceeded" as const
422 }).pipe(Effect.flip)
423
424 assert.strictEqual(error, "maxBytes exceeded")
425 assert.isTrue(stream.destroyed)
426 }))
427
428 it.effect("toString limits the bytes read before decoding", () =>
429 Effect.gen(function*() {
430 const latin1 = yield* NodeStream.toString(() => Readable.from([Buffer.from([0xe9])]), {
431 encoding: "latin1",
432 maxBytes: 1
433 })
434 const utf16le = yield* NodeStream.toString(() => Readable.from([Buffer.from("abcd", "utf16le")]), {
435 encoding: "utf16le",
436 maxBytes: 7,
437 onError: (error) => error
438 }).pipe(Effect.flip)
439
440 assert.strictEqual(latin1, "é")
441 assert.instanceOf(utf16le, NodeStream.MaxBytesError)
442 }))
443
444 it.effect("toString decodes characters split across chunks", () =>
445 Effect.gen(function*() {
446 const chunks = [Buffer.from([0xe2, 0x82]), Buffer.from([0xac, 0xe2])]
447 const text = yield* NodeStream.toString(() => Readable.from(chunks), { maxBytes: 4 })
448
449 assert.strictEqual(text, "€\ufffd")
450 }))
451
452 it.effect("toString limits the bytes read by a stream that already decodes", () =>
453 Effect.gen(function*() {
454 const stream = new Readable({
455 read() {
456 this.push(Buffer.from([0xe9]))
457 this.push(null)
458 }
459 })
460 stream.setEncoding("utf8")
461 const text = yield* NodeStream.toString(() => stream, { encoding: "latin1", maxBytes: 1 })
462
463 assert.strictEqual(text, "é")
464 }))
465
466 it.effect("toString limits object-mode strings by their UTF-8 length", () =>
467 Effect.gen(function*() {
468 const error = yield* NodeStream.toString(() => Readable.from(["a", "b", "c", "d"]), {
469 encoding: "hex",
470 maxBytes: 3,
471 onError: (error) => error
472 }).pipe(Effect.flip)
473
474 assert.instanceOf(error, NodeStream.MaxBytesError)
475 }))
476
477 it.effect("toArrayBuffer enforces a zero maxBytes limit", () =>
478 Effect.gen(function*() {
479 let emitted = false
480 const stream = new Readable({
481 read() {
482 if (emitted) return
483 emitted = true
484 this.push(Buffer.from("a"))
485 }
486 })
487 const error = yield* NodeStream.toArrayBuffer(() => stream, {
488 maxBytes: ByteSize.zero,
489 onError: () => "maxBytes exceeded" as const
490 }).pipe(Effect.flip)
491
492 assert.strictEqual(error, "maxBytes exceeded")
493 assert.isTrue(stream.destroyed)
494 }))
495 })