The best way to control Claude Code, Codex and more from any computer or phone

main
ts 351 lines 13 KB
Raw
1 import * as kafka from "@distilled.cloud/aws/kafka";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import { createPhysicalName } from "alchemy/PhysicalName";
5 import * as Provider from "alchemy/Provider";
6 import { Resource } from "alchemy/Resource";
7 import { createInternalTags, diffTags, hasAlchemyTags } from "alchemy/Tags";
8 import * as Data from "effect/Data";
9 import * as Effect from "effect/Effect";
10 import * as Schedule from "effect/Schedule";
11 import * as Stream from "effect/Stream";
12 import type { Providers } from "../Providers.ts";
13
14 export class ServerlessClusterNotReady extends Data.TaggedError("ServerlessClusterNotReady")<{
15 readonly message: string;
16 readonly cause?: unknown;
17 }> {}
18
19 export class ServerlessClusterUnexpectedResponse extends Data.TaggedError(
20 "ServerlessClusterUnexpectedResponse",
21 )<{
22 readonly message: string;
23 readonly cause?: unknown;
24 }> {}
25
26 export interface ServerlessClusterProps {
27 /**
28 * Name of the MSK Serverless cluster. Must be 1-64 characters. If omitted, a
29 * deterministic physical name is generated. Changing the name replaces the
30 * cluster.
31 */
32 clusterName?: string;
33 /**
34 * VPC subnets the cluster's brokers are reachable from. At least two subnets
35 * in different Availability Zones are required. Changing subnets replaces the
36 * cluster.
37 */
38 subnetIds: string[];
39 /**
40 * VPC security groups that control network access to the cluster. If omitted,
41 * the VPC's default security group is used. Changing security groups replaces
42 * the cluster.
43 */
44 securityGroupIds?: string[];
45 /**
46 * User-defined tags for the cluster.
47 */
48 tags?: Record<string, string>;
49 }
50
51 export interface ServerlessCluster extends Resource<
52 "AWS.Kafka.ServerlessCluster",
53 ServerlessClusterProps,
54 {
55 /** Name of the cluster. */
56 clusterName: string;
57 /** ARN of the cluster. */
58 clusterArn: string;
59 /** Cluster type — always `SERVERLESS` for this resource. */
60 clusterType: string;
61 /** Current state (`CREATING`, `ACTIVE`, `DELETING`, ...). */
62 state: string;
63 /**
64 * SASL/IAM bootstrap broker connection string — the endpoint kafkajs-style
65 * clients connect to using IAM authentication.
66 */
67 bootstrapBrokerStringSaslIam: string | undefined;
68 /** Subnets the cluster is reachable from. */
69 subnetIds: string[];
70 /** Security groups attached to the cluster. */
71 securityGroupIds: string[];
72 /** Tags observed on the cluster. */
73 tags: Record<string, string>;
74 },
75 never,
76 Providers
77 > {}
78
79 /**
80 * An Amazon MSK (Managed Streaming for Apache Kafka) **Serverless** cluster.
81 *
82 * MSK Serverless clusters use IAM authentication exclusively and scale broker
83 * capacity automatically — there is no broker count, instance type, or storage
84 * to configure. They are reachable only from inside a VPC. Creation takes
85 * roughly 5-10 minutes.
86 *
87 * The provisioned (broker-count) MSK cluster is a separate, much slower
88 * (~20-40 minute) resource and is intentionally not modeled here.
89 *
90 * ### Creating a Serverless Cluster
91 * **Example:** Serverless Cluster in a VPC
92 * ```typescript
93 * const cluster = yield* ServerlessCluster("Events", {
94 * subnetIds: [subnetA.subnetId, subnetB.subnetId],
95 * securityGroupIds: [kafkaSecurityGroup.securityGroupId],
96 * });
97 * ```
98 *
99 * ### Consuming from a Lambda Function
100 * **Example:** Wire a topic to a Lambda
101 * ```typescript
102 * yield* Kafka.consumeKafkaTopic(cluster, { topics: ["orders"] }, (records) =>
103 * records.pipe(Stream.runForEach((r) => Effect.log(r.value))),
104 * );
105 * ```
106 *
107 * @resource
108 */
109 export const ServerlessCluster = Resource<ServerlessCluster>("AWS.Kafka.ServerlessCluster");
110
111 const sameStringSet = (
112 a: readonly string[] | undefined,
113 b: readonly string[] | undefined,
114 ): boolean => {
115 const left = [...(a ?? [])].sort();
116 const right = [...(b ?? [])].sort();
117 return left.length === right.length && left.every((v, i) => v === right[i]);
118 };
119
120 const toTagRecord = (
121 tags: { [key: string]: string | undefined } | undefined,
122 ): Record<string, string> =>
123 Object.fromEntries(
124 Object.entries(tags ?? {}).filter(
125 (entry): entry is [string, string] => typeof entry[1] === "string",
126 ),
127 );
128
129 export const ServerlessClusterProvider = () =>
130 Provider.effect(
131 ServerlessCluster,
132 Effect.gen(function* () {
133 const toName = (id: string, props: Partial<ServerlessClusterProps>) =>
134 props.clusterName
135 ? Effect.succeed(props.clusterName)
136 : createPhysicalName({ id, maxLength: 64 });
137
138 // Serverless clusters can only be looked up by ARN via describeClusterV2.
139 // Before we have an ARN (or after a state-persistence failure) we scan
140 // listClustersV2 filtered by name.
141 const findByName = Effect.fn(function* (name: string) {
142 return yield* kafka.listClustersV2
143 .pages({ ClusterNameFilter: name, ClusterTypeFilter: "SERVERLESS" })
144 .pipe(
145 Stream.runCollect,
146 Effect.map((chunk) =>
147 Array.from(chunk)
148 .flatMap((page) => page.ClusterInfoList ?? [])
149 .find((c) => c.ClusterName === name),
150 ),
151 );
152 });
153
154 const describeByArn = Effect.fn(function* (arn: string) {
155 const response = yield* kafka
156 .describeClusterV2({ ClusterArn: arn })
157 .pipe(Effect.catchTag("NotFoundException", () => Effect.undefined));
158 return response?.ClusterInfo;
159 });
160
161 const readBrokers = Effect.fn(function* (arn: string) {
162 return yield* kafka.getBootstrapBrokers({ ClusterArn: arn }).pipe(
163 Effect.map((r) => r.BootstrapBrokerStringSaslIam),
164 // Brokers are only resolvable once the cluster is ACTIVE.
165 Effect.catchTags({
166 BadRequestException: () => Effect.undefined,
167 ConflictException: () => Effect.undefined,
168 }),
169 );
170 });
171
172 // Bounded readiness wait. MSK Serverless create typically completes in
173 // 5-10 minutes; budget ~15 min (60 * 15s).
174 const waitForActive = Effect.fn(function* (arn: string) {
175 const policy = Schedule.max([Schedule.fixed("15 seconds"), Schedule.recurs(60)]);
176 return yield* describeByArn(arn).pipe(
177 Effect.flatMap((cluster) => {
178 if (!cluster) {
179 return Effect.fail(
180 new ServerlessClusterNotReady({ message: `MSK cluster '${arn}' not found` }),
181 );
182 }
183 if (cluster.State === "FAILED") {
184 return Effect.die(
185 new Error(
186 `MSK cluster '${arn}' entered FAILED state: ${cluster.StateInfo?.Message ?? ""}`,
187 ),
188 );
189 }
190 if (cluster.State !== "ACTIVE") {
191 return Effect.fail(
192 new ServerlessClusterNotReady({
193 message: `MSK cluster '${arn}' not active (state: ${cluster.State})`,
194 }),
195 );
196 }
197 return Effect.succeed(cluster);
198 }),
199 Effect.retry({ schedule: policy }),
200 );
201 });
202
203 const toAttrs = Effect.fn(function* (cluster: kafka.Cluster) {
204 if (!cluster.ClusterName || !cluster.ClusterArn) {
205 return yield* new ServerlessClusterUnexpectedResponse({
206 message: `MSK cluster is missing its name or ARN (state: ${cluster.State})`,
207 });
208 }
209 const bootstrap =
210 cluster.State === "ACTIVE" ? yield* readBrokers(cluster.ClusterArn) : undefined;
211 const vpcConfigs = cluster.Serverless?.VpcConfigs ?? [];
212 return {
213 clusterName: cluster.ClusterName,
214 clusterArn: cluster.ClusterArn,
215 clusterType: cluster.ClusterType ?? "SERVERLESS",
216 state: cluster.State ?? "ACTIVE",
217 bootstrapBrokerStringSaslIam: bootstrap,
218 subnetIds: vpcConfigs.flatMap((v) => v.SubnetIds ?? []),
219 securityGroupIds: vpcConfigs.flatMap((v) => v.SecurityGroupIds ?? []),
220 tags: toTagRecord(cluster.Tags),
221 };
222 });
223
224 return {
225 stables: ["clusterName", "clusterArn", "clusterType"],
226
227 diff: Effect.fn(function* ({ id, olds, news }) {
228 if (!isResolved(news)) return undefined;
229 if ((yield* toName(id, olds ?? {})) !== (yield* toName(id, news ?? {}))) {
230 return { action: "replace" } as const;
231 }
232 // VPC configuration is create-only for serverless clusters.
233 if (olds?.subnetIds !== undefined && !sameStringSet(news?.subnetIds, olds.subnetIds)) {
234 return { action: "replace" } as const;
235 }
236 if (
237 olds?.securityGroupIds !== undefined &&
238 !sameStringSet(news?.securityGroupIds, olds.securityGroupIds)
239 ) {
240 return { action: "replace" } as const;
241 }
242 }),
243
244 read: Effect.fn(function* ({ id, olds, output }) {
245 const cluster = output?.clusterArn
246 ? yield* describeByArn(output.clusterArn)
247 : yield* findByName(yield* toName(id, olds ?? {}));
248 if (!cluster?.ClusterArn) return undefined;
249 const attrs = yield* toAttrs(cluster);
250 return (yield* hasAlchemyTags(id, attrs.tags)) ? attrs : Unowned(attrs);
251 }),
252
253 reconcile: Effect.fn(function* ({ id, news, output, session }) {
254 const name = output?.clusterName ?? (yield* toName(id, news));
255 const internalTags = yield* createInternalTags(id);
256 const desiredTags = { ...internalTags, ...news.tags };
257
258 // 1. Observe — cloud state is authoritative. Prefer the cached ARN,
259 // fall back to a name scan (recovery / adoption).
260 let observed = output?.clusterArn
261 ? yield* describeByArn(output.clusterArn)
262 : yield* findByName(name);
263
264 // 2. Ensure — create if missing; tolerate a concurrent create
265 // (ConflictException) by re-scanning for the cluster by name.
266 if (!observed?.ClusterArn) {
267 const created = yield* kafka
268 .createClusterV2({
269 ClusterName: name,
270 Tags: desiredTags,
271 Serverless: {
272 VpcConfigs: [
273 {
274 SubnetIds: news.subnetIds,
275 SecurityGroupIds: news.securityGroupIds,
276 },
277 ],
278 ClientAuthentication: { Sasl: { Iam: { Enabled: true } } },
279 },
280 })
281 .pipe(
282 Effect.map((r) => r.ClusterArn),
283 Effect.catchTag("ConflictException", () =>
284 findByName(name).pipe(Effect.map((c) => c?.ClusterArn)),
285 ),
286 );
287 if (!created) {
288 return yield* Effect.die(
289 new Error(`MSK ServerlessCluster(${id}) could not be created`),
290 );
291 }
292 observed = yield* describeByArn(created);
293 }
294
295 if (!observed?.ClusterArn) {
296 return yield* Effect.die(
297 new Error(`MSK ServerlessCluster(${id}) could not be reconciled`),
298 );
299 }
300 const arn = observed.ClusterArn;
301
302 // Wait for ACTIVE (bounded) so tag sync and broker resolution work.
303 observed = yield* waitForActive(arn);
304
305 // 3. Sync tags — diff against OBSERVED cloud tags so adoption
306 // rewrites ownership tags correctly.
307 const { removed, upsert } = diffTags(toTagRecord(observed.Tags), desiredTags);
308 if (removed.length > 0) {
309 yield* kafka.untagResource({ ResourceArn: arn, TagKeys: removed });
310 }
311 if (upsert.length > 0) {
312 yield* kafka.tagResource({
313 ResourceArn: arn,
314 Tags: Object.fromEntries(upsert.map(({ Key, Value }) => [Key, Value])),
315 });
316 observed = (yield* describeByArn(arn)) ?? observed;
317 }
318
319 // 4. Return fresh attributes.
320 yield* session.note(name);
321 return yield* toAttrs(observed);
322 }),
323
324 delete: Effect.fn(function* ({ output }) {
325 yield* kafka.deleteCluster({ ClusterArn: output.clusterArn }).pipe(
326 Effect.catchTag("NotFoundException", () => Effect.void),
327 // A cluster still CREATING rejects deletion; retry until it can
328 // be deleted (or is gone). Bounded.
329 Effect.retry({
330 while: (e) => e._tag === "BadRequestException",
331 schedule: Schedule.max([Schedule.fixed("15 seconds"), Schedule.recurs(40)]),
332 }),
333 Effect.catchTag("BadRequestException", () => Effect.void),
334 );
335 }),
336
337 list: () =>
338 kafka.listClustersV2.pages({ ClusterTypeFilter: "SERVERLESS" }).pipe(
339 Stream.runCollect,
340 Effect.map((chunk) =>
341 Array.from(chunk).flatMap((page) =>
342 (page.ClusterInfoList ?? []).filter(
343 (c) => c.ClusterName !== undefined && c.ClusterArn !== undefined,
344 ),
345 ),
346 ),
347 Effect.flatMap(Effect.forEach((cluster) => toAttrs(cluster), { concurrency: 4 })),
348 ),
349 };
350 }),
351 );