The best way to control Claude Code, Codex and more from any computer or phone

main
ts 432 lines 14.4 KB
Raw
1 import * as kinesis from "@distilled.cloud/aws/kinesis";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import type { Input } from "alchemy/Input";
5 import { createPhysicalName } from "alchemy/PhysicalName";
6 import * as Provider from "alchemy/Provider";
7 import { Resource } from "alchemy/Resource";
8 import { createInternalTags, diffTags, hasAlchemyTags, type Tags } from "alchemy/Tags";
9 import * as Data from "effect/Data";
10 import * as Effect from "effect/Effect";
11 import * as Schedule from "effect/Schedule";
12 import * as Stream from "effect/Stream";
13 import type { Providers } from "../Providers.ts";
14 import type { StreamArn } from "./Stream.ts";
15
16 export class StreamConsumerUnexpectedResponse extends Data.TaggedError(
17 "StreamConsumerUnexpectedResponse",
18 )<{
19 readonly message: string;
20 readonly cause?: unknown;
21 }> {}
22
23 export type ConsumerName = string;
24
25 export type ConsumerArn = string;
26
27 export type ConsumerStatus = "CREATING" | "DELETING" | "ACTIVE";
28
29 export interface StreamConsumerProps {
30 /**
31 * ARN of the stream that owns the consumer.
32 */
33 streamArn: Input<StreamArn>;
34 /**
35 * Name of the stream consumer.
36 * @default ${app}-${stage}-${id}
37 */
38 consumerName?: string;
39 /**
40 * Tags to associate with the consumer.
41 */
42 tags?: Record<string, string>;
43 }
44
45 export interface StreamConsumer extends Resource<
46 "AWS.Kinesis.StreamConsumer",
47 StreamConsumerProps,
48 {
49 /**
50 * The consumer's physical name.
51 */
52 consumerName: ConsumerName;
53 /**
54 * ARN of the registered consumer.
55 */
56 consumerArn: ConsumerArn;
57 /**
58 * Current lifecycle status of the consumer.
59 */
60 consumerStatus: ConsumerStatus;
61 /**
62 * ARN of the stream the consumer is registered on.
63 */
64 streamArn: StreamArn;
65 /**
66 * When the consumer was registered.
67 */
68 consumerCreationTimestamp: Date;
69 /**
70 * Current tags reported for the consumer.
71 */
72 tags: Record<string, string>;
73 },
74 never,
75 Providers
76 > {}
77
78 /**
79 * A registered Kinesis enhanced fan-out consumer.
80 *
81 * `StreamConsumer` is the canonical lifecycle resource for
82 * `RegisterStreamConsumer` / `DeregisterStreamConsumer`.
83 * ### Creating Consumers
84 * **Example:** Register a Consumer
85 * ```typescript
86 * const consumer = yield* StreamConsumer("AnalyticsConsumer", {
87 * streamArn: stream.streamArn,
88 * });
89 * ```
90 *
91 * @resource
92 */
93 export const StreamConsumer = Resource<StreamConsumer>("AWS.Kinesis.StreamConsumer");
94
95 const createConsumerName = (
96 id: string,
97 props: {
98 consumerName?: string | undefined;
99 },
100 ) =>
101 Effect.gen(function* () {
102 if (props.consumerName) {
103 return props.consumerName;
104 }
105 return yield* createPhysicalName({
106 id,
107 maxLength: 128,
108 });
109 });
110
111 const toTagRecord = (tags: Array<{ Key: string; Value?: string }> | undefined) =>
112 Object.fromEntries(
113 (tags ?? [])
114 .filter((tag): tag is { Key: string; Value: string } => typeof tag.Value === "string")
115 .map((tag) => [tag.Key, tag.Value]),
116 );
117
118 const toAttrs = ({
119 description,
120 tags,
121 }: {
122 description: kinesis.ConsumerDescription;
123 tags: Record<string, string>;
124 }): StreamConsumer["Attributes"] => ({
125 consumerName: description.ConsumerName,
126 consumerArn: description.ConsumerARN,
127 consumerStatus: description.ConsumerStatus as ConsumerStatus,
128 streamArn: description.StreamARN as StreamArn,
129 consumerCreationTimestamp: description.ConsumerCreationTimestamp,
130 tags,
131 });
132
133 const readConsumer = Effect.fn(function* ({
134 streamArn,
135 consumerName,
136 consumerArn,
137 }: {
138 streamArn?: string;
139 consumerName?: string;
140 consumerArn?: string;
141 }) {
142 const response = yield* kinesis
143 .describeStreamConsumer({
144 StreamARN: streamArn,
145 ConsumerName: consumerName,
146 ConsumerARN: consumerArn,
147 })
148 .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.undefined));
149
150 if (!response) {
151 return undefined;
152 }
153
154 const description = response.ConsumerDescription;
155 const tagsResponse = yield* kinesis.listTagsForResource({
156 ResourceARN: description.ConsumerARN,
157 });
158
159 return toAttrs({
160 description,
161 tags: toTagRecord(tagsResponse.Tags),
162 });
163 });
164
165 // Kinesis tells us "in use" via ResourceInUseException, but the consumer
166 // registry is eventually consistent — describeStreamConsumer can briefly
167 // return ResourceNotFoundException for a consumer the registry just
168 // confirmed exists. Poll up to ~10s before giving up.
169 class ConsumerRegistryNotConsistent extends Data.TaggedError("ConsumerRegistryNotConsistent")<{
170 consumerName: string;
171 }> {}
172
173 const adoptExistingConsumer = Effect.fn(function* (streamArn: string, consumerName: string) {
174 return yield* Effect.gen(function* () {
175 const state = yield* readConsumer({
176 streamArn,
177 consumerName,
178 });
179
180 if (!state) {
181 return yield* new ConsumerRegistryNotConsistent({ consumerName });
182 }
183
184 return state;
185 }).pipe(
186 Effect.retry({
187 while: (e) => e._tag === "ConsumerRegistryNotConsistent",
188 schedule: Schedule.max([Schedule.exponential(250), Schedule.recurs(8)]),
189 }),
190 Effect.catchTag("ConsumerRegistryNotConsistent", () =>
191 Effect.fail(
192 new StreamConsumerUnexpectedResponse({
193 message:
194 `consumer ${consumerName} exists but could not be read after ` +
195 `~10s of registry-consistency retries`,
196 }),
197 ),
198 ),
199 );
200 });
201
202 const waitForConsumerStatus = (consumerArn: string, expectedStatus: ConsumerStatus) =>
203 Effect.gen(function* () {
204 yield* Effect.sleep("2 seconds");
205 const response = yield* kinesis.describeStreamConsumer({
206 ConsumerARN: consumerArn,
207 });
208 if (response.ConsumerDescription.ConsumerStatus !== expectedStatus) {
209 return yield* Effect.fail({ _tag: "ConsumerStatusNotReady" as const });
210 }
211 return response.ConsumerDescription;
212 }).pipe(
213 Effect.retry({
214 while: (e: { _tag: string }) =>
215 e._tag === "ConsumerStatusNotReady" || e._tag === "ParseError",
216 schedule: Schedule.max([Schedule.exponential(500), Schedule.recurs(60)]),
217 }),
218 );
219
220 const waitForConsumerDeleted = (consumerArn: string) =>
221 Effect.gen(function* () {
222 yield* kinesis.describeStreamConsumer({
223 ConsumerARN: consumerArn,
224 });
225 return yield* Effect.fail({ _tag: "ConsumerStillExists" as const });
226 }).pipe(
227 Effect.retry({
228 while: (e: { _tag: string }) => e._tag === "ConsumerStillExists" || e._tag === "ParseError",
229 schedule: Schedule.max([Schedule.exponential(500), Schedule.recurs(60)]),
230 }),
231 Effect.catchTag("ResourceNotFoundException", () => Effect.void),
232 );
233
234 export const StreamConsumerProvider = () =>
235 Provider.succeed(StreamConsumer, {
236 stables: ["consumerArn", "consumerName"],
237 read: Effect.fn(function* ({ id, olds, output }) {
238 const consumerName = output?.consumerName ?? (yield* createConsumerName(id, olds ?? {}));
239 const streamArn = output?.streamArn ?? olds?.streamArn;
240 // describeStreamConsumer rejects with InvalidArgumentException unless
241 // either consumerARN, or both streamARN + consumerName, are provided.
242 // If the engine probed for adoption before our upstream Stream was
243 // created we'll have neither — treat that as "doesn't exist".
244 if (typeof streamArn !== "string" && !output?.consumerArn) {
245 return undefined;
246 }
247 const state = yield* readConsumer({
248 streamArn: typeof streamArn === "string" ? streamArn : undefined,
249 consumerName,
250 consumerArn: output?.consumerArn,
251 });
252 if (!state) return undefined;
253 return (yield* hasAlchemyTags(id, state.tags as Tags)) ? state : Unowned(state);
254 }),
255 diff: Effect.fn(function* ({ id, news, olds }) {
256 if (!isResolved(news)) return;
257 const oldConsumerName = yield* createConsumerName(id, olds);
258 const newConsumerName = yield* createConsumerName(id, news);
259 if (oldConsumerName !== newConsumerName) {
260 return { action: "replace" } as const;
261 }
262 if (
263 typeof news.streamArn === "string" &&
264 typeof olds.streamArn === "string" &&
265 news.streamArn !== olds.streamArn
266 ) {
267 return { action: "replace" } as const;
268 }
269 }),
270 reconcile: Effect.fn(function* ({ id, news, output, session }) {
271 const consumerName = output?.consumerName ?? (yield* createConsumerName(id, news));
272 const streamArn = news.streamArn as string;
273 const internalTags = yield* createInternalTags(id);
274 const desiredTags = { ...internalTags, ...news.tags };
275
276 // Observe — fetch live cloud state. `output` is treated as a cache
277 // for the consumer ARN; the consumer's actual existence and tags are
278 // fetched fresh so the reconciler converges regardless of drift,
279 // adoption, or a partially-completed prior run.
280 let state = yield* readConsumer({
281 streamArn,
282 consumerName,
283 consumerArn: output?.consumerArn,
284 });
285
286 // Ensure — register the consumer if it's missing. Tolerate
287 // `ResourceInUseException` as a race with a peer reconciler: a
288 // brief registry-consistency wait, then re-read and continue the
289 // sync path.
290 if (state === undefined) {
291 yield* kinesis
292 .registerStreamConsumer({
293 StreamARN: streamArn,
294 ConsumerName: consumerName,
295 Tags: desiredTags,
296 })
297 .pipe(
298 // The upstream Stream provider waits for ACTIVE via
299 // describeStreamSummary, but the consumer-registry control plane
300 // is eventually consistent against that: registerStreamConsumer
301 // can briefly 404 ("Stream ... not found") for a stream that
302 // describeStreamSummary already reports ACTIVE. Ride out that
303 // propagation window before giving up.
304 Effect.retry({
305 while: (e) => e._tag === "ResourceNotFoundException",
306 schedule: Schedule.max([Schedule.exponential(500), Schedule.recurs(8)]),
307 }),
308 Effect.asVoid,
309 Effect.catchTag("ResourceInUseException", () =>
310 adoptExistingConsumer(streamArn, consumerName).pipe(Effect.asVoid),
311 ),
312 );
313
314 state = yield* adoptExistingConsumer(streamArn, consumerName);
315 yield* waitForConsumerStatus(state.consumerArn, "ACTIVE");
316
317 state = yield* readConsumer({
318 consumerArn: state.consumerArn,
319 streamArn,
320 consumerName,
321 });
322 if (state === undefined) {
323 return yield* new StreamConsumerUnexpectedResponse({
324 message: `failed to read created consumer ${consumerName}`,
325 });
326 }
327 }
328
329 // Sync tags — diff observed cloud tags against desired. Adoption
330 // may bring us a consumer that already has its own tag set; diffing
331 // against `state.tags` (fetched fresh) lets the reconciler converge
332 // ownership without fighting whatever was there before.
333 const { removed, upsert } = diffTags(state.tags, desiredTags);
334
335 if (removed.length > 0) {
336 yield* kinesis.untagResource({
337 ResourceARN: state.consumerArn,
338 TagKeys: removed,
339 });
340 }
341
342 if (upsert.length > 0) {
343 const tagsToAdd: Record<string, string> = {};
344 for (const { Key, Value } of upsert) {
345 tagsToAdd[Key] = Value;
346 }
347 yield* kinesis.tagResource({
348 ResourceARN: state.consumerArn,
349 Tags: tagsToAdd,
350 });
351 }
352
353 // Re-read final state so the returned attributes reflect what's
354 // actually in the cloud after all sync steps.
355 const final = yield* readConsumer({
356 consumerArn: state.consumerArn,
357 });
358 if (!final) {
359 return yield* new StreamConsumerUnexpectedResponse({
360 message: `failed to read reconciled consumer ${consumerName}`,
361 });
362 }
363
364 yield* session.note(final.consumerArn);
365 return final;
366 }),
367 delete: Effect.fn(function* ({ output }) {
368 yield* kinesis
369 .deregisterStreamConsumer({
370 ConsumerARN: output.consumerArn,
371 })
372 .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void));
373
374 yield* waitForConsumerDeleted(output.consumerArn);
375 }),
376 // Consumers are stream-scoped: listStreamConsumers requires a StreamARN.
377 // Fan out — enumerate every stream in the account/region, then list the
378 // consumers registered on each stream, then hydrate tags per consumer.
379 list: Effect.fn(function* () {
380 const streamArns = yield* kinesis.listStreams.pages({}).pipe(
381 Stream.runCollect,
382 Effect.map((chunk) =>
383 Array.from(chunk).flatMap((page) =>
384 (page.StreamSummaries ?? [])
385 .map((summary) => summary.StreamARN)
386 .filter((arn): arn is string => typeof arn === "string"),
387 ),
388 ),
389 );
390
391 const consumers = yield* Effect.forEach(
392 streamArns,
393 (streamArn) =>
394 kinesis.listStreamConsumers.pages({ StreamARN: streamArn }).pipe(
395 Stream.runCollect,
396 Effect.map((chunk) =>
397 Array.from(chunk).flatMap((page) =>
398 (page.Consumers ?? []).map((consumer) => ({
399 streamArn,
400 consumer,
401 })),
402 ),
403 ),
404 // A stream may be deleted mid-enumeration; skip it.
405 Effect.catchTag("ResourceNotFoundException", () =>
406 Effect.succeed([] as { streamArn: string; consumer: kinesis.Consumer }[]),
407 ),
408 ),
409 { concurrency: 10 },
410 ).pipe(Effect.map((rows) => rows.flat()));
411
412 const hydrated = yield* Effect.forEach(
413 consumers,
414 ({ streamArn, consumer }) =>
415 kinesis.listTagsForResource({ ResourceARN: consumer.ConsumerARN }).pipe(
416 Effect.map((tagsResponse): StreamConsumer["Attributes"] => ({
417 consumerName: consumer.ConsumerName,
418 consumerArn: consumer.ConsumerARN,
419 consumerStatus: consumer.ConsumerStatus as ConsumerStatus,
420 streamArn: streamArn as StreamArn,
421 consumerCreationTimestamp: consumer.ConsumerCreationTimestamp,
422 tags: toTagRecord(tagsResponse.Tags),
423 })),
424 // A consumer may be deregistered between list and tag fetch.
425 Effect.catchTag("ResourceNotFoundException", () => Effect.undefined),
426 ),
427 { concurrency: 10 },
428 );
429
430 return hydrated.filter((row): row is StreamConsumer["Attributes"] => row !== undefined);
431 }),
432 });