@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 72 lines 2.74 KB
Raw
1 import type { RuntimeContext } from "alchemy/RuntimeContext";
2 import { makeBatchedSink } from "alchemy/Util/BatchedSink";
3 import type { PipelineRecord } from "cloudflare:pipelines";
4 import type { NonEmptyReadonlyArray } from "effect/Array";
5 import * as Effect from "effect/Effect";
6 import * as Layer from "effect/Layer";
7 import * as Sink from "effect/Sink";
8 import type { Stream } from "./Stream.ts";
9 import { recordSize } from "./StreamCodec.ts";
10 import { StreamSink, type StreamSinkClient } from "./StreamSink.ts";
11 import { StreamSendError, WriteStream, type WriteStreamClient } from "./WriteStream.ts";
12
13 /**
14 * Batch packing for {@link StreamSink}. Internal — not exported from the
15 * Pipelines barrel.
16 */
17
18 /** Pipelines' limit on one ingest request (5 MB), minus the array brackets. */
19 export const MAX_REQUEST_BYTES = 5_000_000 - 2;
20
21 /**
22 * Build the sink over a `WriteStream` client: each chunk is encoded, then
23 * packed into `<= MAX_REQUEST_BYTES` requests (order preserved) and sent
24 * one `sendEncoded` per request.
25 */
26 export const makeStreamSink = <A>(client: WriteStreamClient<A>): StreamSinkClient<A> =>
27 Sink.unwrap(
28 Effect.gen(function* () {
29 // BatchedSink sends without requirements; close over the runtime
30 // context the sink runs in.
31 const context = yield* Effect.context<RuntimeContext>();
32 return makeBatchedSink<PipelineRecord, void, StreamSendError>({
33 maxRecords: Number.MAX_SAFE_INTEGER,
34 maxBytes: MAX_REQUEST_BYTES,
35 sizeOf: recordSize,
36 send: (batch) => client.sendEncoded(batch).pipe(Effect.provideContext(context)),
37 }).pipe(
38 Sink.mapInputArrayEffect((chunk: NonEmptyReadonlyArray<A>) =>
39 client
40 .encode(chunk)
41 .pipe(Effect.map((encoded) => encoded as NonEmptyReadonlyArray<PipelineRecord>)),
42 ),
43 // No `unprocessed` extractor is configured, so retry exhaustion
44 // cannot occur; normalize the type anyway.
45 Sink.mapError((error) =>
46 error._tag === "BatchRetryExhaustedError"
47 ? new StreamSendError({
48 message: `${error.entries.length} records were not ingested`,
49 cause: error,
50 reason: "SendFailed",
51 })
52 : error,
53 ),
54 );
55 }),
56 );
57
58 /**
59 * {@link StreamSink} over whichever {@link WriteStream} implementation is
60 * provided. Composed with each `WriteStream` layer in
61 * `StreamSinkBinding`, `StreamSinkHttp` and `StreamSinkLocal`.
62 */
63 export const StreamSinkFromWriteStream = Layer.effect(
64 StreamSink,
65 Effect.gen(function* () {
66 const writeStream = yield* WriteStream;
67 return Effect.fn(function* (stream: Stream<any>) {
68 const client = yield* writeStream(stream);
69 return makeStreamSink(client);
70 });
71 }),
72 );