The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import { describe, expect, it } from "@alchemy.run/test"; |
| 2 | import * as Data from "effect/Data"; |
| 3 | import * as Effect from "effect/Effect"; |
| 4 | import * as Ref from "effect/Ref"; |
| 5 | import * as Schedule from "effect/Schedule"; |
| 6 | import * as Stream from "effect/Stream"; |
| 7 | import { BatchRetryExhaustedError, makeBatchedSink } from "@/Util/BatchedSink.ts"; |
| 8 | |
| 9 | /** |
| 10 | * Pure tests of the shared batch-sink engine (`Util/BatchedSink.ts`) |
| 11 | * with an injected `send`. Live end-to-end coverage of the three rebased |
| 12 | * sinks (SQS `QueueSink`, SNS `TopicSink`, Kinesis `StreamSink`) lives in |
| 13 | * their fixture tests; partial-failure classification cannot be forced |
| 14 | * deterministically against real AWS, so the retry semantics are proven here. |
| 15 | */ |
| 16 | |
| 17 | class SendFailure extends Data.TaggedError("SendFailure")<{ readonly reason: string }> {} |
| 18 | |
| 19 | interface FakeOut { |
| 20 | /** Indices (relative to the sent batch) reported as transiently unprocessed. */ |
| 21 | readonly unprocessed?: readonly number[]; |
| 22 | /** Indices (relative to the sent batch) reported as permanently rejected. */ |
| 23 | readonly rejected?: readonly number[]; |
| 24 | } |
| 25 | |
| 26 | const byIndex = <In>( |
| 27 | indices: readonly number[] | undefined, |
| 28 | batch: readonly In[], |
| 29 | ): readonly In[] => { |
| 30 | if (indices === undefined || indices.length === 0) { |
| 31 | return []; |
| 32 | } |
| 33 | const set = new Set(indices); |
| 34 | return batch.filter((_, index) => set.has(index)); |
| 35 | }; |
| 36 | |
| 37 | const instant = Schedule.recurs(3); |
| 38 | |
| 39 | describe("makeBatchedSink", { tags: ["unit", "provider:aws", "local"] }, () => { |
| 40 | it.effect("splits a chunk into maxRecords batches preserving order", () => |
| 41 | Effect.gen(function* () { |
| 42 | const sent = yield* Ref.make<(readonly number[])[]>([]); |
| 43 | const sink = makeBatchedSink<number, FakeOut, never>({ |
| 44 | maxRecords: 3, |
| 45 | send: (batch) => Ref.update(sent, (batches) => [...batches, batch]).pipe(Effect.as({})), |
| 46 | }); |
| 47 | |
| 48 | yield* Stream.make(1, 2, 3, 4, 5, 6, 7, 8).pipe(Stream.run(sink)); |
| 49 | |
| 50 | expect(yield* Ref.get(sent)).toEqual([ |
| 51 | [1, 2, 3], |
| 52 | [4, 5, 6], |
| 53 | [7, 8], |
| 54 | ]); |
| 55 | }), |
| 56 | ); |
| 57 | |
| 58 | it.effect("packs greedily by maxBytes using sizeOf", () => |
| 59 | Effect.gen(function* () { |
| 60 | const sent = yield* Ref.make<(readonly string[])[]>([]); |
| 61 | const sink = makeBatchedSink<string, FakeOut, never>({ |
| 62 | maxRecords: 10, |
| 63 | maxBytes: 5, |
| 64 | sizeOf: (record) => record.length, |
| 65 | send: (batch) => Ref.update(sent, (batches) => [...batches, batch]).pipe(Effect.as({})), |
| 66 | }); |
| 67 | |
| 68 | // each record is 2 bytes; 3 records would be 6 > 5, so batches of 2 |
| 69 | yield* Stream.make("aa", "bb", "cc", "dd", "ee").pipe(Stream.run(sink)); |
| 70 | |
| 71 | expect(yield* Ref.get(sent)).toEqual([["aa", "bb"], ["cc", "dd"], ["ee"]]); |
| 72 | }), |
| 73 | ); |
| 74 | |
| 75 | it.effect("re-submits only the unprocessed subset until it converges", () => |
| 76 | Effect.gen(function* () { |
| 77 | const sent = yield* Ref.make<(readonly number[])[]>([]); |
| 78 | const sink = makeBatchedSink<number, FakeOut, never>({ |
| 79 | maxRecords: 10, |
| 80 | retrySchedule: instant, |
| 81 | send: (batch) => |
| 82 | Effect.gen(function* () { |
| 83 | const attempt = (yield* Ref.get(sent)).length; |
| 84 | yield* Ref.update(sent, (batches) => [...batches, batch]); |
| 85 | // first call: report the last two entries as unprocessed; |
| 86 | // second call: everything succeeds. |
| 87 | return attempt === 0 ? { unprocessed: [batch.length - 2, batch.length - 1] } : {}; |
| 88 | }), |
| 89 | unprocessed: (out, batch) => byIndex(out.unprocessed, batch), |
| 90 | }); |
| 91 | |
| 92 | yield* Stream.make(1, 2, 3, 4).pipe(Stream.run(sink)); |
| 93 | |
| 94 | expect(yield* Ref.get(sent)).toEqual([ |
| 95 | [1, 2, 3, 4], |
| 96 | [3, 4], |
| 97 | ]); |
| 98 | }), |
| 99 | ); |
| 100 | |
| 101 | it.effect( |
| 102 | "fails with BatchRetryExhaustedError carrying the stranded entries", |
| 103 | () => |
| 104 | Effect.gen(function* () { |
| 105 | const calls = yield* Ref.make(0); |
| 106 | const sink = makeBatchedSink<number, FakeOut, never>({ |
| 107 | maxRecords: 10, |
| 108 | retrySchedule: Schedule.recurs(2), |
| 109 | send: (batch) => |
| 110 | Ref.update(calls, (n) => n + 1).pipe( |
| 111 | // the last entry is reported unprocessed forever |
| 112 | Effect.as({ unprocessed: [batch.length - 1] }), |
| 113 | ), |
| 114 | unprocessed: (out, batch) => byIndex(out.unprocessed, batch), |
| 115 | }); |
| 116 | |
| 117 | const error = yield* Stream.make(1, 2, 3).pipe(Stream.run(sink), Effect.flip); |
| 118 | |
| 119 | expect(error._tag).toBe("BatchRetryExhaustedError"); |
| 120 | expect((error as BatchRetryExhaustedError<number>).entries).toEqual([3]); |
| 121 | // initial attempt + 2 bounded retries |
| 122 | expect(yield* Ref.get(calls)).toBe(3); |
| 123 | }), |
| 124 | { tags: ["provider:aws:batch"] }, |
| 125 | ); |
| 126 | |
| 127 | it.effect("drops rejected entries without retrying and surfaces them", () => |
| 128 | Effect.gen(function* () { |
| 129 | const calls = yield* Ref.make(0); |
| 130 | const rejected = yield* Ref.make<readonly number[]>([]); |
| 131 | const sink = makeBatchedSink<number, FakeOut, never>({ |
| 132 | maxRecords: 10, |
| 133 | retrySchedule: instant, |
| 134 | send: () => |
| 135 | Ref.update(calls, (n) => n + 1).pipe( |
| 136 | // the middle entry is permanently rejected; nothing is retried |
| 137 | Effect.as({ rejected: [1] } satisfies FakeOut), |
| 138 | ), |
| 139 | unprocessed: (out, batch) => byIndex(out.unprocessed, batch), |
| 140 | rejected: (out, batch) => byIndex(out.rejected, batch), |
| 141 | onRejected: (entries) => Ref.set(rejected, entries), |
| 142 | }); |
| 143 | |
| 144 | yield* Stream.make(1, 2, 3).pipe(Stream.run(sink)); |
| 145 | |
| 146 | expect(yield* Ref.get(calls)).toBe(1); |
| 147 | expect(yield* Ref.get(rejected)).toEqual([2]); |
| 148 | }), |
| 149 | ); |
| 150 | |
| 151 | it.effect("propagates send errors through the sink error channel", () => |
| 152 | Effect.gen(function* () { |
| 153 | const sink = makeBatchedSink<number, FakeOut, SendFailure>({ |
| 154 | maxRecords: 10, |
| 155 | send: () => Effect.fail(new SendFailure({ reason: "boom" })), |
| 156 | }); |
| 157 | |
| 158 | const error = yield* Stream.make(1).pipe(Stream.run(sink), Effect.flip); |
| 159 | |
| 160 | expect(error._tag).toBe("SendFailure"); |
| 161 | }), |
| 162 | ); |
| 163 | }); |