@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 563 lines 19 KB
Raw
1 /**
2 * Adapters between Node streams and Effect streams, channels, and readables.
3 *
4 * This module is the stream boundary for Node APIs. It wraps `Readable` and
5 * `Duplex` values as Effect `Stream`s and `Channel`s, pipes Effect streams
6 * through Node duplex streams, exposes an Effect `Stream` back to Node as a
7 * `Readable`, and collects readable payloads into strings, array buffers, or
8 * `Uint8Array`s with optional byte limits.
9 *
10 * @stability unstable
11 * @since 4.0.0
12 */
13 import * as Arr from "effect/Array"
14 import * as ByteSize from "effect/ByteSize"
15 import * as Cause from "effect/Cause"
16 import * as Channel from "effect/Channel"
17 import * as Context from "effect/Context"
18 import * as Effect from "effect/Effect"
19 import * as Exit from "effect/Exit"
20 import * as Fiber from "effect/Fiber"
21 import { dual, type LazyArg } from "effect/Function"
22 import * as Latch from "effect/Latch"
23 import * as MutableRef from "effect/MutableRef"
24 import * as Pull from "effect/Pull"
25 import * as Scope from "effect/Scope"
26 import * as Stream from "effect/Stream"
27 import type { Duplex } from "node:stream"
28 import { finished, Readable } from "node:stream"
29 import { StringDecoder } from "node:string_decoder"
30 import { destroyStream as destroyReadable } from "./internal/stream.ts"
31 import { pullIntoWritable } from "./NodeSink.ts"
32
33 /**
34 * Error indicating that a buffered Node stream exceeded its byte limit.
35 *
36 * @stability unstable
37 * @category errors
38 * @since 4.0.0
39 */
40 export class MaxBytesError extends Error {
41 readonly _tag = "MaxBytesError"
42 readonly maxBytes: number
43
44 constructor(maxBytes: number) {
45 super("maxBytes exceeded")
46 this.maxBytes = maxBytes
47 }
48 }
49
50 /**
51 * Converts a Node readable stream into an Effect `Stream`, reading chunks with
52 * independent read sizes and batch limits, mapping stream errors with `onError`,
53 * and destroying the readable on completion unless `closeOnDone` is `false`.
54 *
55 * @stability unstable
56 * @category constructors
57 * @since 4.0.0
58 */
59 export const fromReadable = <A = Uint8Array, E = Cause.UnknownError>(options: {
60 readonly evaluate: LazyArg<Readable | NodeJS.ReadableStream>
61 readonly onError?: (error: unknown) => E
62 /**
63 * Maximum number of Node reads per pull (default 4096). Set to Infinity
64 * for unbounded batching. Does not control the size of each Node read.
65 */
66 readonly chunkSize?: number | undefined
67 /**
68 * Size passed to Node's `read(size)`. Omit to use Node's unsized reads.
69 */
70 readonly readSize?: number | undefined
71 readonly closeOnDone?: boolean | undefined
72 }): Stream.Stream<A, E> => Stream.fromChannel(fromReadableChannel<A, E>(options))
73
74 /**
75 * Creates a `Channel` that pulls chunks from a Node readable stream, mapping
76 * errors with `onError` and destroying the readable on completion unless
77 * `closeOnDone` is `false`.
78 *
79 * @stability unstable
80 * @category constructors
81 * @since 4.0.0
82 */
83 export const fromReadableChannel = <A = Uint8Array, E = Cause.UnknownError>(options: {
84 readonly evaluate: LazyArg<Readable | NodeJS.ReadableStream>
85 readonly onError?: (error: unknown) => E
86 /**
87 * Maximum number of Node reads per pull (default 4096). Set to Infinity
88 * for unbounded batching. Does not control the size of each Node read.
89 */
90 readonly chunkSize?: number | undefined
91 /**
92 * Size passed to Node's `read(size)`. Omit to use Node's unsized reads.
93 */
94 readonly readSize?: number | undefined
95 readonly closeOnDone?: boolean | undefined
96 }): Channel.Channel<Arr.NonEmptyReadonlyArray<A>, E> =>
97 Channel.fromTransform((_, scope) =>
98 readableToPullUnsafe({
99 scope,
100 readable: options.evaluate(),
101 onError: options.onError ?? defaultOnError as any,
102 chunkSize: options.chunkSize,
103 readSize: options.readSize,
104 closeOnDone: options.closeOnDone
105 })
106 )
107
108 /**
109 * Creates a `Channel` over a Node `Duplex`, writing upstream chunks with
110 * backpressure while emitting chunks read from the duplex and optionally ending
111 * the writable side when upstream completes.
112 *
113 * @stability unstable
114 * @category constructors
115 * @since 4.0.0
116 */
117 export const fromDuplex = <IE, I = Uint8Array, O = Uint8Array, E = Cause.UnknownError>(
118 options: {
119 readonly evaluate: LazyArg<Duplex>
120 readonly onError?: (error: unknown) => E
121 /**
122 * Maximum number of Node reads per pull (default 4096). Set to Infinity
123 * for unbounded batching. Does not control the size of each Node read.
124 */
125 readonly chunkSize?: number | undefined
126 /**
127 * Size passed to Node's `read(size)`. Omit to use Node's unsized reads.
128 */
129 readonly readSize?: number | undefined
130 readonly endOnDone?: boolean | undefined
131 readonly encoding?: BufferEncoding | undefined
132 }
133 ): Channel.Channel<Arr.NonEmptyReadonlyArray<O>, IE | E, void, Arr.NonEmptyReadonlyArray<I>, IE> =>
134 Channel.fromTransform((upstream, scope) => {
135 const duplex = options.evaluate()
136 const exit = MutableRef.make<Exit.Exit<never, IE | E | Cause.Done> | undefined>(undefined)
137 const latch = Latch.makeUnsafe(false)
138
139 return pullIntoWritable({
140 pull: upstream,
141 writable: duplex,
142 onError: options.onError ?? defaultOnError as any,
143 endOnDone: options.endOnDone,
144 encoding: options.encoding
145 }).pipe(
146 Effect.catchCause((cause) => {
147 if (Pull.isDoneCause(cause)) return Effect.void
148 exit.current = Exit.failCause(cause as Cause.Cause<IE | E | Cause.Done>)
149 latch.openUnsafe()
150 return Effect.void
151 }),
152 Effect.forkIn(scope),
153 Effect.flatMap(() =>
154 readableToPullUnsafe({
155 scope,
156 exit,
157 latch,
158 readable: duplex,
159 onError: options.onError ?? defaultOnError as any,
160 chunkSize: options.chunkSize,
161 readSize: options.readSize
162 })
163 )
164 )
165 })
166
167 /**
168 * Pipes an Effect `Stream` through a Node `Duplex`, writing the stream's
169 * chunks to the duplex and emitting chunks read back from it.
170 *
171 * @stability unstable
172 * @category combinators
173 * @since 4.0.0
174 */
175 export const pipeThroughDuplex: {
176 <B = Uint8Array, E2 = Cause.UnknownError>(
177 options: {
178 readonly evaluate: LazyArg<Duplex>
179 readonly onError?: (error: unknown) => E2
180 /**
181 * Maximum number of Node reads per pull (default 4096). Set to Infinity
182 * for unbounded batching. Does not control the size of each Node read.
183 */
184 readonly chunkSize?: number | undefined
185 /**
186 * Size passed to Node's `read(size)`. Omit to use Node's unsized reads.
187 */
188 readonly readSize?: number | undefined
189 readonly endOnDone?: boolean | undefined
190 readonly encoding?: BufferEncoding | undefined
191 }
192 ): <R, E, A>(self: Stream.Stream<A, E, R>) => Stream.Stream<B, E2 | E, R>
193 <R, E, A, B = Uint8Array, E2 = Cause.UnknownError>(
194 self: Stream.Stream<A, E, R>,
195 options: {
196 readonly evaluate: LazyArg<Duplex>
197 readonly onError?: (error: unknown) => E2
198 /**
199 * Maximum number of Node reads per pull (default 4096). Set to Infinity
200 * for unbounded batching. Does not control the size of each Node read.
201 */
202 readonly chunkSize?: number | undefined
203 /**
204 * Size passed to Node's `read(size)`. Omit to use Node's unsized reads.
205 */
206 readonly readSize?: number | undefined
207 readonly endOnDone?: boolean | undefined
208 readonly encoding?: BufferEncoding | undefined
209 }
210 ): Stream.Stream<B, E | E2, R>
211 } = dual(2, <R, E, A, B = Uint8Array, E2 = Cause.UnknownError>(
212 self: Stream.Stream<A, E, R>,
213 options: {
214 readonly evaluate: LazyArg<Duplex>
215 readonly onError?: (error: unknown) => E2
216 /**
217 * Maximum number of Node reads per pull (default 4096). Set to Infinity
218 * for unbounded batching. Does not control the size of each Node read.
219 */
220 readonly chunkSize?: number | undefined
221 /**
222 * Size passed to Node's `read(size)`. Omit to use Node's unsized reads.
223 */
224 readonly readSize?: number | undefined
225 readonly endOnDone?: boolean | undefined
226 readonly encoding?: BufferEncoding | undefined
227 }
228 ): Stream.Stream<B, E | E2, R> =>
229 Stream.pipeThroughChannelOrFail(
230 self,
231 fromDuplex(options)
232 ))
233
234 /**
235 * Pipes a stream of strings or bytes through a Node `Duplex` using default
236 * options and `Cause.UnknownError` for stream failures.
237 *
238 * @stability unstable
239 * @category combinators
240 * @since 4.0.0
241 */
242 export const pipeThroughSimple: {
243 (
244 duplex: LazyArg<Duplex>
245 ): <R, E>(self: Stream.Stream<string | Uint8Array, E, R>) => Stream.Stream<Uint8Array, E | Cause.UnknownError, R>
246 <R, E>(
247 self: Stream.Stream<string | Uint8Array, E, R>,
248 duplex: LazyArg<Duplex>
249 ): Stream.Stream<Uint8Array, Cause.UnknownError | E, R>
250 } = dual(2, <R, E>(
251 self: Stream.Stream<string | Uint8Array, E, R>,
252 duplex: LazyArg<Duplex>
253 ): Stream.Stream<Uint8Array, Cause.UnknownError | E, R> => pipeThroughDuplex(self, { evaluate: duplex }))
254
255 /**
256 * Converts an Effect `Stream` into a Node `Readable`, using the caller's
257 * Effect context to run the stream and destroying the readable if the stream
258 * fails.
259 *
260 * @stability unstable
261 * @category converting
262 * @since 4.0.0
263 */
264 export const toReadable = <E, R>(stream: Stream.Stream<string | Uint8Array, E, R>): Effect.Effect<Readable, never, R> =>
265 Effect.map(
266 Effect.context<R>(),
267 (context) => new StreamAdapter(context, stream)
268 )
269
270 /**
271 * Converts a service-free Effect `Stream` into a Node `Readable` using an
272 * empty Effect context.
273 *
274 * @stability unstable
275 * @category converting
276 * @since 4.0.0
277 */
278 export const toReadableNever = <E>(stream: Stream.Stream<string | Uint8Array, E, never>): Readable =>
279 new StreamAdapter(
280 Context.empty(),
281 stream
282 )
283
284 /**
285 * Consumes a Node readable stream into a string using the selected encoding,
286 * failing through `onError` on stream errors or when `maxBytes` is exceeded
287 * and destroying the stream on interruption or failure.
288 * `maxBytes` counts the bytes read from the stream. Streams that already emit
289 * strings are counted by encoded string length, which can differ from the
290 * bytes originally read.
291 * Set `destroyOnOverflow` to `false` to drain the remaining data instead of
292 * destroying the stream when the byte limit is exceeded.
293 *
294 * @stability unstable
295 * @category converting
296 * @since 4.0.0
297 */
298 export const toString = <E = Cause.UnknownError>(
299 readable: LazyArg<Readable | NodeJS.ReadableStream>,
300 options?: {
301 readonly onError?: (error: unknown) => E
302 readonly encoding?: BufferEncoding | undefined
303 readonly maxBytes?: ByteSize.Input | undefined
304 readonly destroyOnOverflow?: boolean | undefined
305 }
306 ): Effect.Effect<string, E> => {
307 const maxBytesNumber = toMaxBytes(options?.maxBytes)
308 const onError = options?.onError ?? defaultOnError
309 const encoding = options?.encoding ?? "utf8"
310 return Effect.callback((resume) => {
311 const stream = readable() as Readable
312 // Decoding here keeps `maxBytes` on raw bytes. A stream that already
313 // decodes only emits strings, so switch it to `encoding` instead.
314 // Object-mode strings are not encoded bytes and count as UTF-8.
315 const decodes = stream.readableEncoding !== null
316 if (decodes) stream.setEncoding(encoding)
317 const decoder = new StringDecoder(encoding)
318
319 let string = ""
320 let bytes = 0
321 const cleanup = finished(stream, { readable: true, writable: false }, (err) => {
322 stream.off("data", onData)
323 if (err) {
324 destroyReadable(stream, cleanup)
325 resume(Effect.fail(onError(err) as E))
326 } else {
327 cleanup()
328 resume(Effect.succeed(string + decoder.end()))
329 }
330 })
331 function onData(chunk: string | Uint8Array) {
332 if (typeof chunk === "string") {
333 string += chunk
334 bytes += Buffer.byteLength(chunk, decodes ? encoding : "utf8")
335 } else {
336 string += decoder.write(chunk)
337 bytes += chunk.byteLength
338 }
339 if (maxBytesNumber !== undefined && bytes > maxBytesNumber) {
340 stream.off("data", onData)
341 if (options?.destroyOnOverflow === false) {
342 drainReadable(stream, cleanup)
343 } else {
344 destroyReadable(stream, cleanup)
345 }
346 resume(Effect.fail(onError(new MaxBytesError(maxBytesNumber)) as E))
347 }
348 }
349 stream.on("data", onData)
350 return Effect.sync(() => {
351 stream.off("data", onData)
352 destroyReadable(stream, cleanup)
353 })
354 })
355 }
356
357 /**
358 * Consumes a Node readable stream into an `ArrayBuffer`, failing through
359 * `onError` on stream errors or when `maxBytes` is exceeded and destroying the
360 * stream on interruption or failure.
361 * Set `destroyOnOverflow` to `false` to drain the remaining data instead of
362 * destroying the stream when the byte limit is exceeded.
363 *
364 * @stability unstable
365 * @category converting
366 * @since 4.0.0
367 */
368 export const toArrayBuffer = <E = Cause.UnknownError>(
369 readable: LazyArg<Readable | NodeJS.ReadableStream>,
370 options?: {
371 readonly onError?: (error: unknown) => E
372 readonly maxBytes?: ByteSize.Input | undefined
373 readonly destroyOnOverflow?: boolean | undefined
374 }
375 ): Effect.Effect<ArrayBuffer, E> => {
376 const maxBytesNumber = toMaxBytes(options?.maxBytes)
377 const onError = options?.onError ?? defaultOnError
378 return Effect.callback((resume) => {
379 const stream = readable() as Readable
380 const buffers: Array<Uint8Array> = []
381 let bytes = 0
382 const cleanup = finished(stream, { readable: true, writable: false }, (err) => {
383 stream.off("data", onData)
384 if (err) {
385 destroyReadable(stream, cleanup)
386 return resume(Effect.fail(onError(err) as E))
387 }
388 cleanup()
389 const buffer = buffers.length === 1 ? buffers[0] : Buffer.concat(buffers)
390 if (buffer.byteOffset === 0 && buffer.buffer.byteLength === buffer.byteLength) {
391 return resume(Effect.succeed(buffer.buffer as ArrayBuffer))
392 }
393 resume(
394 Effect.succeed(buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength) as ArrayBuffer)
395 )
396 })
397 function onData(chunk: Uint8Array) {
398 buffers.push(chunk)
399 bytes += chunk.length
400 if (maxBytesNumber !== undefined && bytes > maxBytesNumber) {
401 stream.off("data", onData)
402 if (options?.destroyOnOverflow === false) {
403 drainReadable(stream, cleanup)
404 } else {
405 destroyReadable(stream, cleanup)
406 }
407 resume(Effect.fail(onError(new MaxBytesError(maxBytesNumber)) as E))
408 }
409 }
410 stream.on("data", onData)
411 return Effect.sync(() => {
412 stream.off("data", onData)
413 destroyReadable(stream, cleanup)
414 })
415 })
416 }
417
418 /**
419 * Consumes a Node readable stream into a `Uint8Array`, using the same error
420 * mapping and `maxBytes` handling as `toArrayBuffer`.
421 *
422 * @stability unstable
423 * @category converting
424 * @since 4.0.0
425 */
426 export const toUint8Array = <E = Cause.UnknownError>(
427 readable: LazyArg<Readable | NodeJS.ReadableStream>,
428 options?: {
429 readonly onError?: (error: unknown) => E
430 readonly maxBytes?: ByteSize.Input | undefined
431 readonly destroyOnOverflow?: boolean | undefined
432 }
433 ): Effect.Effect<Uint8Array, E> => Effect.map(toArrayBuffer(readable, options), (buffer) => new Uint8Array(buffer))
434
435 // ----------------------------------------------------------------------------
436 // internal
437 // ----------------------------------------------------------------------------
438
439 // Keep consuming after the consumer's listeners are removed, so the producer
440 // can finish and any late error stays handled.
441 const drainReadable = (stream: Readable, cleanup: () => void): void => {
442 cleanup()
443 const done = finished(stream, { readable: true, writable: false }, () => done())
444 stream.resume()
445 }
446
447 const readableToPullUnsafe = <A, E>(options: {
448 readonly scope: Scope.Scope
449 readonly exit?: MutableRef.MutableRef<Exit.Exit<never, E | Cause.Done> | undefined> | undefined
450 readonly latch?: Latch.Latch | undefined
451 readonly readable: Readable | NodeJS.ReadableStream
452 readonly onError: (error: unknown) => E
453 readonly chunkSize: number | undefined
454 readonly readSize: number | undefined
455 readonly closeOnDone?: boolean | undefined
456 }) => {
457 const readable = options.readable as Readable
458 const chunkSize = options.chunkSize ?? 4096
459
460 const closeOnDone = options.closeOnDone ?? true
461 const exit = options.exit ?? MutableRef.make(undefined)
462 const latch = options.latch ?? Latch.makeUnsafe(false)
463 function onReadable() {
464 latch.openUnsafe()
465 }
466 readable.on("readable", onReadable)
467 const cleanup = finished(readable, { readable: true, writable: false }, (error) => {
468 exit.current ??= Exit.fail(error ? options.onError(error) : Cause.Done())
469 latch.openUnsafe()
470 })
471
472 const pull = Effect.suspend(function loop(): Pull.Pull<Arr.NonEmptyReadonlyArray<A>, E> {
473 let item = options.readable.read(options.readSize) as A | null
474 if (item === null) {
475 if (exit.current) {
476 return exit.current
477 }
478 if (readable.readableEnded) {
479 return Effect.fail(Cause.Done())
480 }
481 latch.closeUnsafe()
482 return Effect.flatMap(latch.await, loop)
483 }
484 const chunk = Arr.of(item as A)
485 // read() can synchronously refill the buffer, so null is not a batch bound.
486 while (chunk.length < chunkSize) {
487 item = options.readable.read(options.readSize)
488 if (item === null) break
489 chunk.push(item)
490 }
491 return Effect.succeed(chunk)
492 })
493
494 return Effect.as(
495 Scope.addFinalizer(
496 options.scope,
497 Effect.sync(() => {
498 readable.off("readable", onReadable)
499 if (closeOnDone) {
500 destroyReadable(readable, cleanup)
501 } else {
502 cleanup()
503 }
504 })
505 ),
506 pull
507 )
508 }
509
510 class StreamAdapter<E, R> extends Readable {
511 private readonly readLatch: Latch.Latch
512 private fiber: Fiber.Fiber<void, E> | undefined = undefined
513
514 constructor(
515 context: Context.Context<R>,
516 stream: Stream.Stream<Uint8Array | string, E, R>
517 ) {
518 super({})
519 this.readLatch = Latch.makeUnsafe(false)
520 this.fiber = Stream.runForEachArray(stream, (chunk) =>
521 this.readLatch.whenOpen(Effect.sync(() => {
522 this.readLatch.closeUnsafe()
523 for (let i = 0; i < chunk.length; i++) {
524 const item = chunk[i]
525 if (typeof item === "string") {
526 this.push(item, "utf8")
527 } else {
528 this.push(item)
529 }
530 }
531 }))).pipe(
532 this.readLatch.whenOpen,
533 Effect.provideContext(context),
534 Effect.runFork
535 )
536 this.fiber.addObserver((exit) => {
537 this.fiber = undefined
538 if (Exit.isSuccess(exit)) {
539 this.push(null)
540 } else {
541 this.destroy(Cause.squash(exit.cause) as any)
542 }
543 })
544 }
545
546 override _read(_size: number): void {
547 this.readLatch.openUnsafe()
548 }
549
550 override _destroy(error: Error | null, callback: (error?: Error | null | undefined) => void): void {
551 if (!this.fiber) {
552 return callback(error)
553 }
554 Effect.runFork(Fiber.interrupt(this.fiber)).addObserver((exit) => {
555 callback(exit._tag === "Failure" ? Cause.squash(exit.cause) as any : error)
556 })
557 }
558 }
559
560 const defaultOnError = (error: unknown): Cause.UnknownError => new Cause.UnknownError(error)
561
562 const toMaxBytes = (maxBytes: ByteSize.Input | undefined): number | undefined =>
563 maxBytes === undefined || maxBytes === Infinity ? undefined : Number(ByteSize.fromInputUnsafe(maxBytes))