The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 06:00 UTC
| 1 | import type { RuntimeContext } from "alchemy/RuntimeContext"; |
| 2 | import { makeBatchedSink } from "alchemy/Util/BatchedSink"; |
| 3 | import type { PipelineRecord } from "cloudflare:pipelines"; |
| 4 | import type { NonEmptyReadonlyArray } from "effect/Array"; |
| 5 | import * as Effect from "effect/Effect"; |
| 6 | import * as Layer from "effect/Layer"; |
| 7 | import * as Sink from "effect/Sink"; |
| 8 | import type { Stream } from "./Stream.ts"; |
| 9 | import { recordSize } from "./StreamCodec.ts"; |
| 10 | import { StreamSink, type StreamSinkClient } from "./StreamSink.ts"; |
| 11 | import { StreamSendError, WriteStream, type WriteStreamClient } from "./WriteStream.ts"; |
| 12 | |
| 13 | /** |
| 14 | * Batch packing for {@link StreamSink}. Internal — not exported from the |
| 15 | * Pipelines barrel. |
| 16 | */ |
| 17 | |
| 18 | /** Pipelines' limit on one ingest request (5 MB), minus the array brackets. */ |
| 19 | export const MAX_REQUEST_BYTES = 5_000_000 - 2; |
| 20 | |
| 21 | /** |
| 22 | * Build the sink over a `WriteStream` client: each chunk is encoded, then |
| 23 | * packed into `<= MAX_REQUEST_BYTES` requests (order preserved) and sent |
| 24 | * one `sendEncoded` per request. |
| 25 | */ |
| 26 | export const makeStreamSink = <A>(client: WriteStreamClient<A>): StreamSinkClient<A> => |
| 27 | Sink.unwrap( |
| 28 | Effect.gen(function* () { |
| 29 | // BatchedSink sends without requirements; close over the runtime |
| 30 | // context the sink runs in. |
| 31 | const context = yield* Effect.context<RuntimeContext>(); |
| 32 | return makeBatchedSink<PipelineRecord, void, StreamSendError>({ |
| 33 | maxRecords: Number.MAX_SAFE_INTEGER, |
| 34 | maxBytes: MAX_REQUEST_BYTES, |
| 35 | sizeOf: recordSize, |
| 36 | send: (batch) => client.sendEncoded(batch).pipe(Effect.provideContext(context)), |
| 37 | }).pipe( |
| 38 | Sink.mapInputArrayEffect((chunk: NonEmptyReadonlyArray<A>) => |
| 39 | client |
| 40 | .encode(chunk) |
| 41 | .pipe(Effect.map((encoded) => encoded as NonEmptyReadonlyArray<PipelineRecord>)), |
| 42 | ), |
| 43 | // No `unprocessed` extractor is configured, so retry exhaustion |
| 44 | // cannot occur; normalize the type anyway. |
| 45 | Sink.mapError((error) => |
| 46 | error._tag === "BatchRetryExhaustedError" |
| 47 | ? new StreamSendError({ |
| 48 | message: `${error.entries.length} records were not ingested`, |
| 49 | cause: error, |
| 50 | reason: "SendFailed", |
| 51 | }) |
| 52 | : error, |
| 53 | ), |
| 54 | ); |
| 55 | }), |
| 56 | ); |
| 57 | |
| 58 | /** |
| 59 | * {@link StreamSink} over whichever {@link WriteStream} implementation is |
| 60 | * provided. Composed with each `WriteStream` layer in |
| 61 | * `StreamSinkBinding`, `StreamSinkHttp` and `StreamSinkLocal`. |
| 62 | */ |
| 63 | export const StreamSinkFromWriteStream = Layer.effect( |
| 64 | StreamSink, |
| 65 | Effect.gen(function* () { |
| 66 | const writeStream = yield* WriteStream; |
| 67 | return Effect.fn(function* (stream: Stream<any>) { |
| 68 | const client = yield* writeStream(stream); |
| 69 | return makeStreamSink(client); |
| 70 | }); |
| 71 | }), |
| 72 | ); |