@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 163 lines 5.84 KB
Raw
1 import { describe, expect, it } from "@alchemy.run/test";
2 import * as Data from "effect/Data";
3 import * as Effect from "effect/Effect";
4 import * as Ref from "effect/Ref";
5 import * as Schedule from "effect/Schedule";
6 import * as Stream from "effect/Stream";
7 import { BatchRetryExhaustedError, makeBatchedSink } from "@/Util/BatchedSink.ts";
8
9 /**
10 * Pure tests of the shared batch-sink engine (`Util/BatchedSink.ts`)
11 * with an injected `send`. Live end-to-end coverage of the three rebased
12 * sinks (SQS `QueueSink`, SNS `TopicSink`, Kinesis `StreamSink`) lives in
13 * their fixture tests; partial-failure classification cannot be forced
14 * deterministically against real AWS, so the retry semantics are proven here.
15 */
16
17 class SendFailure extends Data.TaggedError("SendFailure")<{ readonly reason: string }> {}
18
19 interface FakeOut {
20 /** Indices (relative to the sent batch) reported as transiently unprocessed. */
21 readonly unprocessed?: readonly number[];
22 /** Indices (relative to the sent batch) reported as permanently rejected. */
23 readonly rejected?: readonly number[];
24 }
25
26 const byIndex = <In>(
27 indices: readonly number[] | undefined,
28 batch: readonly In[],
29 ): readonly In[] => {
30 if (indices === undefined || indices.length === 0) {
31 return [];
32 }
33 const set = new Set(indices);
34 return batch.filter((_, index) => set.has(index));
35 };
36
37 const instant = Schedule.recurs(3);
38
39 describe("makeBatchedSink", { tags: ["unit", "provider:aws", "local"] }, () => {
40 it.effect("splits a chunk into maxRecords batches preserving order", () =>
41 Effect.gen(function* () {
42 const sent = yield* Ref.make<(readonly number[])[]>([]);
43 const sink = makeBatchedSink<number, FakeOut, never>({
44 maxRecords: 3,
45 send: (batch) => Ref.update(sent, (batches) => [...batches, batch]).pipe(Effect.as({})),
46 });
47
48 yield* Stream.make(1, 2, 3, 4, 5, 6, 7, 8).pipe(Stream.run(sink));
49
50 expect(yield* Ref.get(sent)).toEqual([
51 [1, 2, 3],
52 [4, 5, 6],
53 [7, 8],
54 ]);
55 }),
56 );
57
58 it.effect("packs greedily by maxBytes using sizeOf", () =>
59 Effect.gen(function* () {
60 const sent = yield* Ref.make<(readonly string[])[]>([]);
61 const sink = makeBatchedSink<string, FakeOut, never>({
62 maxRecords: 10,
63 maxBytes: 5,
64 sizeOf: (record) => record.length,
65 send: (batch) => Ref.update(sent, (batches) => [...batches, batch]).pipe(Effect.as({})),
66 });
67
68 // each record is 2 bytes; 3 records would be 6 > 5, so batches of 2
69 yield* Stream.make("aa", "bb", "cc", "dd", "ee").pipe(Stream.run(sink));
70
71 expect(yield* Ref.get(sent)).toEqual([["aa", "bb"], ["cc", "dd"], ["ee"]]);
72 }),
73 );
74
75 it.effect("re-submits only the unprocessed subset until it converges", () =>
76 Effect.gen(function* () {
77 const sent = yield* Ref.make<(readonly number[])[]>([]);
78 const sink = makeBatchedSink<number, FakeOut, never>({
79 maxRecords: 10,
80 retrySchedule: instant,
81 send: (batch) =>
82 Effect.gen(function* () {
83 const attempt = (yield* Ref.get(sent)).length;
84 yield* Ref.update(sent, (batches) => [...batches, batch]);
85 // first call: report the last two entries as unprocessed;
86 // second call: everything succeeds.
87 return attempt === 0 ? { unprocessed: [batch.length - 2, batch.length - 1] } : {};
88 }),
89 unprocessed: (out, batch) => byIndex(out.unprocessed, batch),
90 });
91
92 yield* Stream.make(1, 2, 3, 4).pipe(Stream.run(sink));
93
94 expect(yield* Ref.get(sent)).toEqual([
95 [1, 2, 3, 4],
96 [3, 4],
97 ]);
98 }),
99 );
100
101 it.effect(
102 "fails with BatchRetryExhaustedError carrying the stranded entries",
103 () =>
104 Effect.gen(function* () {
105 const calls = yield* Ref.make(0);
106 const sink = makeBatchedSink<number, FakeOut, never>({
107 maxRecords: 10,
108 retrySchedule: Schedule.recurs(2),
109 send: (batch) =>
110 Ref.update(calls, (n) => n + 1).pipe(
111 // the last entry is reported unprocessed forever
112 Effect.as({ unprocessed: [batch.length - 1] }),
113 ),
114 unprocessed: (out, batch) => byIndex(out.unprocessed, batch),
115 });
116
117 const error = yield* Stream.make(1, 2, 3).pipe(Stream.run(sink), Effect.flip);
118
119 expect(error._tag).toBe("BatchRetryExhaustedError");
120 expect((error as BatchRetryExhaustedError<number>).entries).toEqual([3]);
121 // initial attempt + 2 bounded retries
122 expect(yield* Ref.get(calls)).toBe(3);
123 }),
124 { tags: ["provider:aws:batch"] },
125 );
126
127 it.effect("drops rejected entries without retrying and surfaces them", () =>
128 Effect.gen(function* () {
129 const calls = yield* Ref.make(0);
130 const rejected = yield* Ref.make<readonly number[]>([]);
131 const sink = makeBatchedSink<number, FakeOut, never>({
132 maxRecords: 10,
133 retrySchedule: instant,
134 send: () =>
135 Ref.update(calls, (n) => n + 1).pipe(
136 // the middle entry is permanently rejected; nothing is retried
137 Effect.as({ rejected: [1] } satisfies FakeOut),
138 ),
139 unprocessed: (out, batch) => byIndex(out.unprocessed, batch),
140 rejected: (out, batch) => byIndex(out.rejected, batch),
141 onRejected: (entries) => Ref.set(rejected, entries),
142 });
143
144 yield* Stream.make(1, 2, 3).pipe(Stream.run(sink));
145
146 expect(yield* Ref.get(calls)).toBe(1);
147 expect(yield* Ref.get(rejected)).toEqual([2]);
148 }),
149 );
150
151 it.effect("propagates send errors through the sink error channel", () =>
152 Effect.gen(function* () {
153 const sink = makeBatchedSink<number, FakeOut, SendFailure>({
154 maxRecords: 10,
155 send: () => Effect.fail(new SendFailure({ reason: "boom" })),
156 });
157
158 const error = yield* Stream.make(1).pipe(Stream.run(sink), Effect.flip);
159
160 expect(error._tag).toBe("SendFailure");
161 }),
162 );
163 });