The best way to control Claude Code, Codex and more from any computer or phone

main
ts 995 lines 36.9 KB
Raw
1 import * as firehose from "@distilled.cloud/aws/firehose";
2 import * as iam from "@distilled.cloud/aws/iam";
3 import { Unowned } from "alchemy/AdoptPolicy";
4 import { isResolved } from "alchemy/Diff";
5 import { createPhysicalName } from "alchemy/PhysicalName";
6 import * as Provider from "alchemy/Provider";
7 import { Resource } from "alchemy/Resource";
8 import {
9 createInternalTags,
10 createTagsList,
11 diffTags,
12 hasAlchemyTags,
13 type Tags,
14 } from "alchemy/Tags";
15 import { toWireSeconds } from "alchemy/Util/Duration";
16 import * as Data from "effect/Data";
17 import type * as Duration from "effect/Duration";
18 import * as Effect from "effect/Effect";
19 import * as Schedule from "effect/Schedule";
20 import { AWSEnvironment, type AccountID } from "../Environment.ts";
21 import type { PolicyStatement } from "../IAM/Policy.ts";
22 import type { Providers } from "../Providers.ts";
23 import type { RegionID } from "../Region.ts";
24
25 export type DeliveryStreamName = string;
26 export type DeliveryStreamArn =
27 `arn:aws:firehose:${RegionID}:${AccountID}:deliverystream/${DeliveryStreamName}`;
28
29 export type DeliveryStreamStatus =
30 | "CREATING"
31 | "CREATING_FAILED"
32 | "DELETING"
33 | "DELETING_FAILED"
34 | "ACTIVE";
35
36 export type DeliveryStreamSourceType = "DirectPut" | "KinesisStreamAsSource";
37
38 export type CompressionFormat = "UNCOMPRESSED" | "GZIP" | "ZIP" | "Snappy" | "HADOOP_SNAPPY";
39
40 export interface KinesisStreamSourceProps {
41 /**
42 * ARN of the Kinesis Data Stream that feeds the delivery stream.
43 * Changing the source stream replaces the delivery stream.
44 */
45 kinesisStreamArn: string;
46 /**
47 * ARN of the IAM role Firehose assumes to read from the Kinesis stream.
48 * @default a role is auto-created granting kinesis:DescribeStream,
49 * kinesis:GetShardIterator, kinesis:GetRecords and kinesis:ListShards on
50 * the source stream.
51 */
52 roleArn?: string;
53 }
54
55 export interface S3DestinationProps {
56 /**
57 * ARN of the destination S3 bucket.
58 */
59 bucketArn: string;
60 /**
61 * ARN of the IAM role Firehose assumes to write to the bucket.
62 * @default a role is auto-created granting s3:PutObject, s3:GetObject,
63 * s3:ListBucket, s3:GetBucketLocation, s3:AbortMultipartUpload and
64 * s3:ListBucketMultipartUploads on the bucket.
65 */
66 roleArn?: string;
67 /**
68 * Prefix prepended to delivered S3 object keys.
69 */
70 prefix?: string;
71 /**
72 * Prefix for objects that failed delivery or transformation.
73 */
74 errorOutputPrefix?: string;
75 /**
76 * Buffering interval before flushing to S3 — e.g. `"1 minute"` or
77 * `Duration.seconds(60)`. Sent to AWS as whole seconds; valid values range
78 * from 0 (zero buffering) to 900 seconds.
79 * @default "300 seconds"
80 */
81 bufferingInterval?: Duration.Input;
82 /**
83 * Buffering size in MiB before flushing to S3.
84 * Valid values range from 1 to 128.
85 * @default 5
86 */
87 bufferingSizeInMBs?: number;
88 /**
89 * Compression format applied to delivered objects.
90 * @default "UNCOMPRESSED"
91 */
92 compressionFormat?: CompressionFormat;
93 }
94
95 export type DeliveryStreamEncryptionKeyType = "AWS_OWNED_CMK" | "CUSTOMER_MANAGED_CMK";
96
97 export type DeliveryStreamEncryptionStatus =
98 | "ENABLED"
99 | "ENABLING"
100 | "ENABLING_FAILED"
101 | "DISABLED"
102 | "DISABLING"
103 | "DISABLING_FAILED";
104
105 export interface DeliveryStreamEncryptionProps {
106 /**
107 * Which key to use for server-side encryption: the AWS-owned CMK that
108 * Firehose manages for you, or a customer-managed KMS key (`keyArn`
109 * required).
110 */
111 keyType: DeliveryStreamEncryptionKeyType;
112 /**
113 * ARN of the customer-managed KMS key. Required when `keyType` is
114 * `CUSTOMER_MANAGED_CMK`; must be omitted for `AWS_OWNED_CMK`.
115 */
116 keyArn?: string;
117 }
118
119 export interface DeliveryStreamProps {
120 /**
121 * Name of the delivery stream. Changing the name replaces the stream.
122 * @default ${app}-${id}-${stage}-${instanceId}
123 */
124 deliveryStreamName?: string;
125 /**
126 * Kinesis Data Stream source. When omitted the stream is `DirectPut` and
127 * producers write via `PutRecord` / `PutRecordBatch`. Adding, removing or
128 * changing the source replaces the delivery stream.
129 */
130 source?: KinesisStreamSourceProps;
131 /**
132 * S3 destination configuration (Firehose "extended S3" destination).
133 * Destination settings update in place via `UpdateDestination`.
134 */
135 destination: S3DestinationProps;
136 /**
137 * Server-side encryption (SSE) for records at rest inside the delivery
138 * stream. Only supported for `DirectPut` streams — streams with a Kinesis
139 * source inherit encryption from the source stream. Adding, changing or
140 * removing encryption updates the stream in place via
141 * `StartDeliveryStreamEncryption` / `StopDeliveryStreamEncryption`.
142 * @default no server-side encryption
143 */
144 encryption?: DeliveryStreamEncryptionProps;
145 /**
146 * Tags to associate with the delivery stream.
147 */
148 tags?: Record<string, string>;
149 }
150
151 export interface DeliveryStream extends Resource<
152 "AWS.Firehose.DeliveryStream",
153 DeliveryStreamProps,
154 {
155 /**
156 * The delivery stream's physical name.
157 */
158 deliveryStreamName: DeliveryStreamName;
159 /**
160 * ARN of the delivery stream.
161 */
162 deliveryStreamArn: DeliveryStreamArn;
163 /**
164 * Current lifecycle status of the delivery stream.
165 */
166 deliveryStreamStatus: DeliveryStreamStatus;
167 /**
168 * Source type of the delivery stream.
169 */
170 deliveryStreamType: DeliveryStreamSourceType;
171 /**
172 * Current version ID of the delivery stream configuration.
173 */
174 versionId: string;
175 /**
176 * ID of the (single) destination attached to the stream.
177 */
178 destinationId: string | undefined;
179 /**
180 * ARN of the destination S3 bucket.
181 */
182 bucketArn: string;
183 /**
184 * ARN of the IAM role Firehose assumes to write to the destination.
185 */
186 roleArn: string;
187 /**
188 * Name of the auto-created IAM role, when one was synthesized for this
189 * stream. `undefined` when the caller supplied every role ARN.
190 */
191 roleName: string | undefined;
192 /**
193 * ARN of the source Kinesis stream, for `KinesisStreamAsSource` streams.
194 */
195 kinesisStreamArn: string | undefined;
196 /**
197 * Prefix prepended to delivered S3 object keys.
198 */
199 prefix: string | undefined;
200 /**
201 * Prefix for objects that failed delivery.
202 */
203 errorOutputPrefix: string | undefined;
204 /**
205 * Buffering interval in seconds currently configured on the destination.
206 */
207 bufferingIntervalInSeconds: number | undefined;
208 /**
209 * Buffering size in MiB currently configured on the destination.
210 */
211 bufferingSizeInMBs: number | undefined;
212 /**
213 * Compression format currently configured on the destination.
214 */
215 compressionFormat: CompressionFormat | undefined;
216 /**
217 * Server-side encryption status reported by the stream — `ENABLED` when
218 * SSE is active, `DISABLED` when it is not.
219 */
220 encryptionStatus: DeliveryStreamEncryptionStatus | undefined;
221 /**
222 * Key type used for server-side encryption, when enabled.
223 */
224 encryptionKeyType: DeliveryStreamEncryptionKeyType | undefined;
225 /**
226 * ARN of the customer-managed KMS key used for server-side encryption,
227 * when `encryptionKeyType` is `CUSTOMER_MANAGED_CMK`.
228 */
229 encryptionKeyArn: string | undefined;
230 /**
231 * Current tags reported for the delivery stream.
232 */
233 tags: Record<string, string>;
234 },
235 never,
236 Providers
237 > {}
238
239 /**
240 * An Amazon Data Firehose delivery stream that buffers records and delivers
241 * them to an S3 bucket.
242 *
243 * `DeliveryStream` owns the stream's lifecycle and mutable destination
244 * configuration (buffering hints, compression, prefixes, tags). The stream is
245 * `DirectPut` by default — producers write with `PutRecord` /
246 * `PutRecordBatch` — or it can drain an existing Kinesis Data Stream via the
247 * `source` prop. Unless you supply role ARNs, an IAM role is auto-created
248 * granting Firehose write access to the destination bucket (and read access
249 * to the source stream when one is configured).
250 * ### Creating Delivery Streams
251 * **Example:** DirectPut stream delivering to S3
252 * ```typescript
253 * import * as AWS from "@alchemy.run/aws";
254 *
255 * const bucket = yield* AWS.S3.Bucket("DataLake");
256 * const stream = yield* AWS.Firehose.DeliveryStream("Events", {
257 * destination: {
258 * bucketArn: bucket.bucketArn,
259 * },
260 * });
261 * ```
262 *
263 * **Example:** Tuned buffering and compression
264 * ```typescript
265 * const stream = yield* AWS.Firehose.DeliveryStream("Events", {
266 * destination: {
267 * bucketArn: bucket.bucketArn,
268 * prefix: "events/",
269 * errorOutputPrefix: "errors/",
270 * bufferingInterval: "1 minute",
271 * bufferingSizeInMBs: 1,
272 * compressionFormat: "GZIP",
273 * },
274 * });
275 * ```
276 *
277 * **Example:** Server-side encryption at rest
278 * ```typescript
279 * const stream = yield* AWS.Firehose.DeliveryStream("Events", {
280 * destination: { bucketArn: bucket.bucketArn },
281 * encryption: { keyType: "AWS_OWNED_CMK" },
282 * });
283 * ```
284 *
285 * **Example:** Kinesis Data Stream as source
286 * ```typescript
287 * const source = yield* AWS.Kinesis.Stream("Clickstream");
288 * const stream = yield* AWS.Firehose.DeliveryStream("ClickstreamArchive", {
289 * source: { kinesisStreamArn: source.streamArn },
290 * destination: { bucketArn: bucket.bucketArn },
291 * });
292 * ```
293 *
294 * ### Runtime Producers
295 * Bind producer operations in the init phase and use them in runtime
296 * handlers. Records are buffered by Firehose and appear in S3 after the
297 * buffering interval elapses.
298 *
299 * **Example:** Put a record from a handler
300 * ```typescript
301 * // init
302 * const putRecord = yield* AWS.Firehose.PutRecord(stream);
303 *
304 * return {
305 * fetch: Effect.gen(function* () {
306 * // runtime
307 * const response = yield* putRecord({
308 * Record: { Data: new TextEncoder().encode("hello\n") },
309 * });
310 * return HttpServerResponse.json({ recordId: response.RecordId });
311 * }),
312 * };
313 * ```
314 *
315 * **Example:** Put a batch of records
316 * ```typescript
317 * // init
318 * const putRecordBatch = yield* AWS.Firehose.PutRecordBatch(stream);
319 *
320 * // runtime
321 * const response = yield* putRecordBatch({
322 * Records: lines.map((line) => ({
323 * Data: new TextEncoder().encode(`${line}\n`),
324 * })),
325 * });
326 * ```
327 *
328 * @resource
329 */
330 export const DeliveryStream = Resource<DeliveryStream>("AWS.Firehose.DeliveryStream");
331
332 /**
333 * The delivery stream entered `CREATING_FAILED` — AWS never recovers this
334 * state; the stream must be deleted and recreated.
335 */
336 export class DeliveryStreamCreateFailed extends Data.TaggedError("DeliveryStreamCreateFailed")<{
337 readonly deliveryStreamName: string;
338 readonly details: string | undefined;
339 }> {}
340
341 /**
342 * Validation error raised before any AWS call when the props are invalid.
343 */
344 export class DeliveryStreamValidationError extends Data.TaggedError(
345 "DeliveryStreamValidationError",
346 )<{
347 readonly message: string;
348 }> {}
349
350 /**
351 * Server-side encryption reached a terminal failure state
352 * (`ENABLING_FAILED` / `DISABLING_FAILED`) while converging to the desired
353 * encryption configuration.
354 */
355 export class DeliveryStreamEncryptionFailed extends Data.TaggedError(
356 "DeliveryStreamEncryptionFailed",
357 )<{
358 readonly deliveryStreamName: string;
359 readonly status: string;
360 readonly details: string | undefined;
361 }> {}
362
363 class DeliveryStreamNotActive extends Data.TaggedError("DeliveryStreamNotActive")<{
364 readonly status: string;
365 }> {}
366
367 class DeliveryStreamEncryptionPending extends Data.TaggedError("DeliveryStreamEncryptionPending") {}
368
369 class DeliveryStreamStillExists extends Data.TaggedError("DeliveryStreamStillExists") {}
370
371 const defaultBufferingIntervalInSeconds = 300;
372 const defaultBufferingSizeInMBs = 5;
373 const defaultCompressionFormat = "UNCOMPRESSED" as const;
374
375 const createDeliveryStreamName = (id: string, props: { deliveryStreamName?: string | undefined }) =>
376 Effect.gen(function* () {
377 if (props.deliveryStreamName) {
378 return props.deliveryStreamName;
379 }
380 return yield* createPhysicalName({ id, maxLength: 64 });
381 });
382
383 /**
384 * Data-FIRST `Effect.retry(self, options)` wrapped with an explicit return
385 * annotation — inlining `Effect.retry` with an options object in provider
386 * lifecycle code leaves `Retry.Return`'s conditional type unresolved in the
387 * provider's inferred layer type, which TypeScript's declaration emit widens
388 * to an `unknown` R — poisoning `AWS.providers()` for every downstream
389 * consumer (see `retryThroughDeletionWindow` in SecretsManager/Secret.ts).
390 *
391 * Retries `InvalidArgumentException`s that mention the IAM role — a freshly
392 * created role takes a few seconds to become assumable by the Firehose
393 * service principal, and `CreateDeliveryStream` surfaces that propagation
394 * window as a role-related `InvalidArgumentException`.
395 */
396 const retryThroughRolePropagation = <A, E extends { _tag: string }, R>(
397 self: Effect.Effect<A, E, R>,
398 ): Effect.Effect<A, E, R> =>
399 Effect.retry(self, {
400 while: (e) =>
401 e._tag === "InvalidArgumentException" &&
402 /role|assume|trust/i.test((e as { message?: string }).message ?? ""),
403 schedule: Schedule.max([Schedule.fixed("2 seconds"), Schedule.recurs(15)]),
404 });
405
406 /**
407 * Retries `ResourceInUseException` (stream still `CREATING`, or a concurrent
408 * operation in flight) on a bounded schedule. Explicitly annotated for the
409 * same declaration-emit reason as {@link retryThroughRolePropagation}.
410 */
411 const retryWhileInUse = <A, E extends { _tag: string }, R>(
412 self: Effect.Effect<A, E, R>,
413 ): Effect.Effect<A, E, R> =>
414 Effect.retry(self, {
415 while: (e) => e._tag === "ResourceInUseException",
416 schedule: Schedule.max([Schedule.fixed("2 seconds"), Schedule.recurs(30)]),
417 });
418
419 /**
420 * Retries `ConcurrentModificationException` from `UpdateDestination` — the
421 * read-modify-write cycle re-reads the fresh `VersionId` on every attempt.
422 * Explicitly annotated for the same declaration-emit reason as
423 * {@link retryThroughRolePropagation}.
424 */
425 const retryConcurrentModification = <A, E extends { _tag: string }, R>(
426 self: Effect.Effect<A, E, R>,
427 ): Effect.Effect<A, E, R> =>
428 Effect.retry(self, {
429 while: (e) => e._tag === "ConcurrentModificationException",
430 schedule: Schedule.max([Schedule.fixed("2 seconds"), Schedule.recurs(5)]),
431 });
432
433 const toTagRecord = (tags: ReadonlyArray<{ Key: string; Value?: string }> | undefined) =>
434 Object.fromEntries(
435 (tags ?? [])
436 .filter((tag): tag is { Key: string; Value: string } => typeof tag.Value === "string")
437 .map((tag) => [tag.Key, tag.Value]),
438 );
439
440 const toAttrs = ({
441 description,
442 tags,
443 roleName,
444 }: {
445 description: firehose.DeliveryStreamDescription;
446 tags: Record<string, string>;
447 roleName: string | undefined;
448 }): DeliveryStream["Attributes"] => {
449 const destination = description.Destinations[0];
450 const s3 = destination?.ExtendedS3DestinationDescription;
451 const encryption = description.DeliveryStreamEncryptionConfiguration;
452 return {
453 deliveryStreamName: description.DeliveryStreamName,
454 deliveryStreamArn: description.DeliveryStreamARN as DeliveryStreamArn,
455 deliveryStreamStatus: description.DeliveryStreamStatus as DeliveryStreamStatus,
456 deliveryStreamType: description.DeliveryStreamType as DeliveryStreamSourceType,
457 versionId: description.VersionId,
458 destinationId: destination?.DestinationId,
459 bucketArn: s3?.BucketARN ?? "",
460 roleArn: s3?.RoleARN ?? "",
461 roleName,
462 kinesisStreamArn: description.Source?.KinesisStreamSourceDescription?.KinesisStreamARN,
463 prefix: s3?.Prefix || undefined,
464 errorOutputPrefix: s3?.ErrorOutputPrefix || undefined,
465 bufferingIntervalInSeconds: s3?.BufferingHints?.IntervalInSeconds,
466 bufferingSizeInMBs: s3?.BufferingHints?.SizeInMBs,
467 compressionFormat: s3?.CompressionFormat as CompressionFormat | undefined,
468 encryptionStatus: encryption?.Status as DeliveryStreamEncryptionStatus | undefined,
469 encryptionKeyType:
470 encryption?.Status === "ENABLED"
471 ? (encryption.KeyType as DeliveryStreamEncryptionKeyType | undefined)
472 : undefined,
473 encryptionKeyArn: encryption?.Status === "ENABLED" ? encryption.KeyARN : undefined,
474 tags,
475 };
476 };
477
478 const describeDeliveryStream = Effect.fn(function* (deliveryStreamName: string) {
479 const response = yield* firehose
480 .describeDeliveryStream({ DeliveryStreamName: deliveryStreamName })
481 .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.undefined));
482 return response?.DeliveryStreamDescription;
483 });
484
485 const readDeliveryStream = Effect.fn(function* ({
486 deliveryStreamName,
487 roleName,
488 }: {
489 deliveryStreamName: string;
490 roleName: string | undefined;
491 }) {
492 const description = yield* describeDeliveryStream(deliveryStreamName);
493 if (!description) {
494 return undefined;
495 }
496 // The stream can vanish between the describe above and the tag read (e.g.
497 // a concurrent destroy) — a typed `ResourceNotFoundException` here just
498 // means it's gone.
499 const tagsResponse = yield* firehose
500 .listTagsForDeliveryStream({ DeliveryStreamName: deliveryStreamName })
501 .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.undefined));
502 if (!tagsResponse) {
503 return undefined;
504 }
505 return toAttrs({
506 description,
507 tags: toTagRecord(tagsResponse.Tags),
508 roleName,
509 });
510 });
511
512 // Firehose activates fast (usually well under a minute); 2s × 45 bounds the
513 // wait at 90s.
514 const waitForDeliveryStreamActive = (deliveryStreamName: string) =>
515 Effect.gen(function* () {
516 const description = yield* describeDeliveryStream(deliveryStreamName);
517 if (!description) {
518 return yield* new DeliveryStreamNotActive({ status: "MISSING" });
519 }
520 if (description.DeliveryStreamStatus === "CREATING_FAILED") {
521 return yield* new DeliveryStreamCreateFailed({
522 deliveryStreamName,
523 details: description.FailureDescription?.Details,
524 });
525 }
526 if (description.DeliveryStreamStatus !== "ACTIVE") {
527 return yield* new DeliveryStreamNotActive({
528 status: description.DeliveryStreamStatus,
529 });
530 }
531 return description;
532 }).pipe(
533 Effect.retry({
534 while: (e: { _tag: string }) => e._tag === "DeliveryStreamNotActive",
535 schedule: Schedule.max([Schedule.fixed("2 seconds"), Schedule.recurs(45)]),
536 }),
537 );
538
539 // SSE transitions (ENABLING → ENABLED, DISABLING → DISABLED) settle in
540 // seconds for the AWS-owned CMK; 2s × 45 bounds the wait at 90s. Returns the
541 // terminal encryption configuration (or `undefined` when the stream has
542 // never been encrypted).
543 const waitForEncryptionSettled = (deliveryStreamName: string) =>
544 Effect.gen(function* () {
545 const description = yield* describeDeliveryStream(deliveryStreamName);
546 const encryption = description?.DeliveryStreamEncryptionConfiguration;
547 if (encryption?.Status === "ENABLING" || encryption?.Status === "DISABLING") {
548 return yield* new DeliveryStreamEncryptionPending();
549 }
550 return encryption;
551 }).pipe(
552 Effect.retry({
553 while: (e: { _tag: string }) => e._tag === "DeliveryStreamEncryptionPending",
554 schedule: Schedule.max([Schedule.fixed("2 seconds"), Schedule.recurs(45)]),
555 }),
556 );
557
558 // Deletion is usually quick but AWS documents that DELETING can linger;
559 // 3s × 50 bounds the wait at 150s.
560 const waitForDeliveryStreamDeleted = (deliveryStreamName: string) =>
561 Effect.gen(function* () {
562 const description = yield* describeDeliveryStream(deliveryStreamName);
563 if (description !== undefined) {
564 return yield* new DeliveryStreamStillExists();
565 }
566 }).pipe(
567 Effect.retry({
568 while: (e: { _tag: string }) => e._tag === "DeliveryStreamStillExists",
569 schedule: Schedule.max([Schedule.fixed("3 seconds"), Schedule.recurs(50)]),
570 }),
571 );
572
573 const deleteRoleIfExists = Effect.fn(function* (roleName: string) {
574 yield* iam
575 .deleteRolePolicy({ RoleName: roleName, PolicyName: roleName })
576 .pipe(Effect.catchTag("NoSuchEntityException", () => Effect.void));
577 yield* iam
578 .deleteRole({ RoleName: roleName })
579 .pipe(Effect.catchTag("NoSuchEntityException", () => Effect.void));
580 });
581
582 export const DeliveryStreamProvider = () =>
583 Provider.effect(
584 DeliveryStream,
585 Effect.gen(function* () {
586 const createRoleName = (id: string) => createPhysicalName({ id, maxLength: 64 });
587
588 // Ensure the synthesized IAM role exists and its inline policy matches
589 // the desired destination/source access. `createRole` tolerates the
590 // already-exists race; `putRolePolicy` is an idempotent upsert, so the
591 // policy re-converges on every reconcile (e.g. after a bucket change).
592 const ensureRole = Effect.fn(function* ({
593 id,
594 roleName,
595 bucketArn,
596 kinesisStreamArn,
597 }: {
598 id: string;
599 roleName: string;
600 bucketArn: string;
601 kinesisStreamArn: string | undefined;
602 }) {
603 const tags = yield* createInternalTags(id);
604 yield* iam
605 .createRole({
606 RoleName: roleName,
607 AssumeRolePolicyDocument: JSON.stringify({
608 Version: "2012-10-17",
609 Statement: [
610 {
611 Effect: "Allow",
612 Principal: { Service: "firehose.amazonaws.com" },
613 Action: "sts:AssumeRole",
614 },
615 ],
616 }),
617 Tags: createTagsList(tags),
618 })
619 .pipe(
620 Effect.catchTag("EntityAlreadyExistsException", () =>
621 iam.getRole({ RoleName: roleName }),
622 ),
623 );
624
625 const statements: PolicyStatement[] = [
626 {
627 Effect: "Allow",
628 Action: [
629 "s3:AbortMultipartUpload",
630 "s3:GetBucketLocation",
631 "s3:GetObject",
632 "s3:ListBucket",
633 "s3:ListBucketMultipartUploads",
634 "s3:PutObject",
635 ],
636 Resource: [bucketArn, `${bucketArn}/*`],
637 },
638 ];
639 if (kinesisStreamArn) {
640 statements.push({
641 Effect: "Allow",
642 Action: [
643 "kinesis:DescribeStream",
644 "kinesis:GetShardIterator",
645 "kinesis:GetRecords",
646 "kinesis:ListShards",
647 ],
648 Resource: [kinesisStreamArn],
649 });
650 }
651
652 yield* iam.putRolePolicy({
653 RoleName: roleName,
654 PolicyName: roleName,
655 PolicyDocument: JSON.stringify({
656 Version: "2012-10-17",
657 Statement: statements,
658 }),
659 });
660 });
661
662 return DeliveryStream.Provider.of({
663 stables: ["deliveryStreamName", "deliveryStreamArn", "deliveryStreamType", "roleName"],
664
665 // Enumerate every delivery stream in the ambient account/region.
666 // `listDeliveryStreams` pages via ExclusiveStartDeliveryStreamName +
667 // HasMoreDeliveryStreams; hydrate each name into the `read` shape
668 // with bounded concurrency, dropping streams that vanish mid-flight.
669 list: () =>
670 Effect.gen(function* () {
671 const names: string[] = [];
672 let exclusiveStart: string | undefined;
673 while (true) {
674 const page = yield* firehose.listDeliveryStreams({
675 Limit: 100,
676 ExclusiveStartDeliveryStreamName: exclusiveStart,
677 });
678 names.push(...page.DeliveryStreamNames);
679 if (!page.HasMoreDeliveryStreams || page.DeliveryStreamNames.length === 0) {
680 break;
681 }
682 exclusiveStart = names[names.length - 1];
683 }
684
685 const hydrated = yield* Effect.forEach(
686 names,
687 (deliveryStreamName) =>
688 readDeliveryStream({ deliveryStreamName, roleName: undefined }),
689 { concurrency: 10 },
690 );
691
692 return hydrated.filter(
693 (attrs): attrs is DeliveryStream["Attributes"] => attrs !== undefined,
694 );
695 }),
696
697 read: Effect.fn(function* ({ id, olds, output }) {
698 const deliveryStreamName =
699 output?.deliveryStreamName ?? (yield* createDeliveryStreamName(id, olds ?? {}));
700 const state = yield* readDeliveryStream({
701 deliveryStreamName,
702 roleName: output?.roleName,
703 });
704 if (!state) return undefined;
705 return (yield* hasAlchemyTags(id, state.tags as Tags)) ? state : Unowned(state);
706 }),
707
708 diff: Effect.fn(function* ({ id, news, olds }) {
709 if (!isResolved(news)) return undefined;
710 const oldName = yield* createDeliveryStreamName(id, olds ?? {});
711 const newName = yield* createDeliveryStreamName(id, news ?? {});
712 if (oldName !== newName) {
713 return { action: "replace" } as const;
714 }
715 // The source (DirectPut vs KinesisStreamAsSource, and which stream)
716 // is immutable — any change replaces the delivery stream.
717 const oldSource = olds?.source;
718 const newSource = news?.source;
719 if (
720 (oldSource === undefined) !== (newSource === undefined) ||
721 oldSource?.kinesisStreamArn !== newSource?.kinesisStreamArn
722 ) {
723 return { action: "replace" } as const;
724 }
725 }),
726
727 reconcile: Effect.fn(function* ({ id, news, output, session }) {
728 const { accountId } = yield* AWSEnvironment.current;
729
730 if (!news?.destination?.bucketArn) {
731 return yield* new DeliveryStreamValidationError({
732 message: `DeliveryStream "${id}" requires destination.bucketArn`,
733 });
734 }
735 if (news.encryption && news.source) {
736 return yield* new DeliveryStreamValidationError({
737 message: `DeliveryStream "${id}" cannot enable server-side encryption on a KinesisStreamAsSource stream — encryption is inherited from the source Kinesis stream`,
738 });
739 }
740 if (news.encryption?.keyType === "CUSTOMER_MANAGED_CMK" && !news.encryption.keyArn) {
741 return yield* new DeliveryStreamValidationError({
742 message: `DeliveryStream "${id}" requires encryption.keyArn when keyType is CUSTOMER_MANAGED_CMK`,
743 });
744 }
745
746 const deliveryStreamName =
747 output?.deliveryStreamName ?? (yield* createDeliveryStreamName(id, news));
748 const internalTags = yield* createInternalTags(id);
749 const desiredTags = { ...news.tags, ...internalTags };
750
751 // Synthesize the IAM role when the caller didn't supply one for
752 // the destination (and/or the Kinesis source).
753 const needsRole =
754 !news.destination.roleArn || (news.source !== undefined && !news.source.roleArn);
755 const roleName = needsRole
756 ? (output?.roleName ?? (yield* createRoleName(id)))
757 : undefined;
758 const synthesizedRoleArn = roleName
759 ? `arn:aws:iam::${accountId}:role/${roleName}`
760 : undefined;
761 const destinationRoleArn = news.destination.roleArn ?? synthesizedRoleArn!;
762 const sourceRoleArn = news.source
763 ? (news.source.roleArn ?? synthesizedRoleArn!)
764 : undefined;
765
766 if (roleName) {
767 yield* ensureRole({
768 id,
769 roleName,
770 bucketArn: news.destination.bucketArn,
771 kinesisStreamArn: news.source?.kinesisStreamArn,
772 });
773 }
774
775 // Observe — cloud state is authoritative; `output` is only a cache
776 // for the physical name.
777 let observed = yield* describeDeliveryStream(deliveryStreamName);
778
779 // A stream stuck in CREATING_FAILED / DELETING_FAILED can never
780 // recover — converge by deleting it and recreating below.
781 if (
782 observed &&
783 (observed.DeliveryStreamStatus === "CREATING_FAILED" ||
784 observed.DeliveryStreamStatus === "DELETING_FAILED")
785 ) {
786 yield* session.note(
787 `Delivery stream ${deliveryStreamName} is ${observed.DeliveryStreamStatus}; deleting and recreating`,
788 );
789 yield* firehose
790 .deleteDeliveryStream({
791 DeliveryStreamName: deliveryStreamName,
792 AllowForceDelete: true,
793 })
794 .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void));
795 yield* waitForDeliveryStreamDeleted(deliveryStreamName);
796 observed = undefined;
797 }
798
799 const desiredBufferingHints = {
800 IntervalInSeconds:
801 toWireSeconds(news.destination.bufferingInterval) ??
802 defaultBufferingIntervalInSeconds,
803 SizeInMBs: news.destination.bufferingSizeInMBs ?? defaultBufferingSizeInMBs,
804 };
805 const desiredCompression = news.destination.compressionFormat ?? defaultCompressionFormat;
806
807 // Ensure — create the stream if it's missing. Tolerate the
808 // already-exists race (`ResourceInUseException`) and retry through
809 // the fresh role's IAM propagation window.
810 if (observed === undefined) {
811 yield* retryThroughRolePropagation(
812 firehose
813 .createDeliveryStream({
814 DeliveryStreamName: deliveryStreamName,
815 DeliveryStreamType: news.source ? "KinesisStreamAsSource" : "DirectPut",
816 KinesisStreamSourceConfiguration: news.source
817 ? {
818 KinesisStreamARN: news.source.kinesisStreamArn,
819 RoleARN: sourceRoleArn!,
820 }
821 : undefined,
822 DeliveryStreamEncryptionConfigurationInput: news.encryption
823 ? {
824 KeyType: news.encryption.keyType,
825 KeyARN: news.encryption.keyArn,
826 }
827 : undefined,
828 ExtendedS3DestinationConfiguration: {
829 RoleARN: destinationRoleArn,
830 BucketARN: news.destination.bucketArn,
831 Prefix: news.destination.prefix,
832 ErrorOutputPrefix: news.destination.errorOutputPrefix,
833 BufferingHints: desiredBufferingHints,
834 CompressionFormat: desiredCompression,
835 },
836 Tags: createTagsList(desiredTags),
837 })
838 .pipe(Effect.catchTag("ResourceInUseException", () => Effect.void)),
839 );
840 yield* session.note(`Creating delivery stream ${deliveryStreamName}...`);
841 }
842
843 // Both a fresh create and a crashed prior run land here in
844 // `CREATING` — wait for `ACTIVE` before syncing (bounded).
845 const active = yield* waitForDeliveryStreamActive(deliveryStreamName);
846
847 // Sync destination settings — diff observed against desired and
848 // apply only the delta via UpdateDestination (read-modify-write on
849 // VersionId, retried through ConcurrentModificationException).
850 const syncDestination = Effect.gen(function* () {
851 const description = yield* describeDeliveryStream(deliveryStreamName);
852 if (!description) return;
853 const destination = description.Destinations[0];
854 const s3 = destination?.ExtendedS3DestinationDescription;
855 if (!destination || !s3) return;
856
857 const delta =
858 s3.RoleARN !== destinationRoleArn ||
859 s3.BucketARN !== news.destination.bucketArn ||
860 (s3.Prefix ?? "") !== (news.destination.prefix ?? "") ||
861 (s3.ErrorOutputPrefix ?? "") !== (news.destination.errorOutputPrefix ?? "") ||
862 s3.BufferingHints.IntervalInSeconds !== desiredBufferingHints.IntervalInSeconds ||
863 s3.BufferingHints.SizeInMBs !== desiredBufferingHints.SizeInMBs ||
864 s3.CompressionFormat !== desiredCompression;
865 if (!delta) return;
866
867 yield* firehose.updateDestination({
868 DeliveryStreamName: deliveryStreamName,
869 CurrentDeliveryStreamVersionId: description.VersionId,
870 DestinationId: destination.DestinationId,
871 ExtendedS3DestinationUpdate: {
872 RoleARN: destinationRoleArn,
873 BucketARN: news.destination.bucketArn,
874 Prefix: news.destination.prefix ?? "",
875 ErrorOutputPrefix: news.destination.errorOutputPrefix ?? "",
876 BufferingHints: desiredBufferingHints,
877 CompressionFormat: desiredCompression,
878 },
879 });
880 yield* session.note(`Updated destination settings for ${deliveryStreamName}`);
881 });
882 yield* retryConcurrentModification(syncDestination);
883
884 // Sync server-side encryption — settle any in-flight transition
885 // (a crashed prior run may have left ENABLING/DISABLING), diff the
886 // observed terminal state against the desired config, and apply
887 // Start/Stop only on a delta. Both operations are async; the
888 // bounded settle-wait converges them. A terminal *_FAILED status
889 // also mismatches the desired state, so the op is re-applied.
890 const observedEncryption = yield* waitForEncryptionSettled(deliveryStreamName);
891 const encryptionEnabled = observedEncryption?.Status === "ENABLED";
892 if (news.encryption) {
893 const keyMatches =
894 encryptionEnabled &&
895 observedEncryption?.KeyType === news.encryption.keyType &&
896 (news.encryption.keyType === "AWS_OWNED_CMK" ||
897 observedEncryption?.KeyARN === news.encryption.keyArn);
898 if (!keyMatches) {
899 yield* retryWhileInUse(
900 firehose.startDeliveryStreamEncryption({
901 DeliveryStreamName: deliveryStreamName,
902 DeliveryStreamEncryptionConfigurationInput: {
903 KeyType: news.encryption.keyType,
904 KeyARN: news.encryption.keyArn,
905 },
906 }),
907 );
908 const settled = yield* waitForEncryptionSettled(deliveryStreamName);
909 if (settled?.Status !== "ENABLED") {
910 return yield* new DeliveryStreamEncryptionFailed({
911 deliveryStreamName,
912 status: settled?.Status ?? "UNKNOWN",
913 details: settled?.FailureDescription?.Details,
914 });
915 }
916 yield* session.note(
917 `Enabled SSE (${news.encryption.keyType}) on ${deliveryStreamName}`,
918 );
919 }
920 } else if (encryptionEnabled) {
921 yield* retryWhileInUse(
922 firehose.stopDeliveryStreamEncryption({
923 DeliveryStreamName: deliveryStreamName,
924 }),
925 );
926 const settled = yield* waitForEncryptionSettled(deliveryStreamName);
927 if (settled !== undefined && settled.Status !== "DISABLED") {
928 return yield* new DeliveryStreamEncryptionFailed({
929 deliveryStreamName,
930 status: settled.Status ?? "UNKNOWN",
931 details: settled.FailureDescription?.Details,
932 });
933 }
934 yield* session.note(`Disabled SSE on ${deliveryStreamName}`);
935 }
936
937 // Sync tags — diff against OBSERVED cloud tags (adoption may bring
938 // a stream with foreign tags), never olds/output.
939 const observedTagsResponse = yield* firehose
940 .listTagsForDeliveryStream({
941 DeliveryStreamName: deliveryStreamName,
942 })
943 .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.undefined));
944 const observedTags = toTagRecord(observedTagsResponse?.Tags);
945 const { removed, upsert } = diffTags(observedTags, desiredTags);
946 if (removed.length > 0) {
947 yield* firehose.untagDeliveryStream({
948 DeliveryStreamName: deliveryStreamName,
949 TagKeys: removed,
950 });
951 }
952 if (upsert.length > 0) {
953 yield* firehose.tagDeliveryStream({
954 DeliveryStreamName: deliveryStreamName,
955 Tags: upsert,
956 });
957 }
958
959 yield* session.note(active.DeliveryStreamARN);
960
961 // Return fresh attributes reflecting post-sync cloud state.
962 const final = yield* readDeliveryStream({
963 deliveryStreamName,
964 roleName,
965 });
966 if (!final) {
967 return yield* new DeliveryStreamValidationError({
968 message: `failed to read reconciled delivery stream ${deliveryStreamName}`,
969 });
970 }
971 return final;
972 }),
973
974 delete: Effect.fn(function* ({ output }) {
975 // A stream in CREATING can't be deleted — retry through the window
976 // (bounded). Already-gone is success.
977 yield* retryWhileInUse(
978 firehose
979 .deleteDeliveryStream({
980 DeliveryStreamName: output.deliveryStreamName,
981 AllowForceDelete: true,
982 })
983 .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void)),
984 );
985 yield* waitForDeliveryStreamDeleted(output.deliveryStreamName);
986
987 // Remove the synthesized IAM role (idempotent — it may already be
988 // gone from a previous partial delete).
989 if (output.roleName) {
990 yield* deleteRoleIfExists(output.roleName);
991 }
992 }),
993 });
994 }),
995 );