@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 463 lines 15.3 KB
Raw
1 import * as osis from "@distilled.cloud/aws/osis";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import { createPhysicalName } from "alchemy/PhysicalName";
5 import * as Provider from "alchemy/Provider";
6 import { Resource } from "alchemy/Resource";
7 import { createInternalTags, diffTags, hasAlchemyTags } from "alchemy/Tags";
8 import * as Data from "effect/Data";
9 import * as Effect from "effect/Effect";
10 import * as Stream from "effect/Stream";
11 import type { Providers } from "../Providers.ts";
12 import {
13 jsonEquals,
14 readPipelineTags,
15 retryWhilePipelineConflict,
16 waitForPipelineSettled,
17 } from "./internal.ts";
18
19 export class OSISPipelineUnexpectedResponse extends Data.TaggedError(
20 "OSISPipelineUnexpectedResponse",
21 )<{
22 readonly message: string;
23 readonly cause?: unknown;
24 }> {}
25
26 export interface PipelineLogPublishingOptions {
27 /**
28 * Whether pipeline logs are published to CloudWatch Logs.
29 */
30 isLoggingEnabled?: boolean;
31 /**
32 * CloudWatch Logs destination. The log group name must start with
33 * `/aws/vendedlogs/`.
34 */
35 cloudWatchLogDestination?: {
36 /** Name of the CloudWatch Logs log group, e.g. `/aws/vendedlogs/OpenSearchIngestion/my-pipeline`. */
37 logGroup: string;
38 };
39 }
40
41 export interface PipelineBufferOptions {
42 /**
43 * Whether persistent buffering is enabled for the pipeline.
44 */
45 persistentBufferEnabled: boolean;
46 }
47
48 export interface PipelineEncryptionAtRestOptions {
49 /**
50 * ARN of the customer-managed KMS key used to encrypt buffer data.
51 */
52 kmsKeyArn: string;
53 }
54
55 export interface PipelineVpcOptions {
56 /**
57 * Subnet IDs the pipeline's VPC endpoint is placed into. Changing VPC
58 * options replaces the pipeline.
59 */
60 subnetIds: string[];
61 /**
62 * Security group IDs applied to the pipeline's VPC endpoint.
63 */
64 securityGroupIds?: string[];
65 }
66
67 export interface PipelineProps {
68 /**
69 * Name of the pipeline. 3-28 characters; lowercase letters, numbers, and
70 * hyphens. If omitted, a deterministic physical name is generated.
71 * Changing the name replaces the pipeline.
72 */
73 pipelineName?: string;
74 /**
75 * Minimum number of Ingestion OpenSearch Compute Units (OCUs) the pipeline
76 * scales down to. Minimum 1.
77 */
78 minUnits: number;
79 /**
80 * Maximum number of Ingestion OpenSearch Compute Units (OCUs) the pipeline
81 * scales up to.
82 */
83 maxUnits: number;
84 /**
85 * Data Prepper pipeline configuration in YAML (must start with
86 * `version: "2"`). Blueprints for common source/sink topologies are
87 * available via the `listPipelineBlueprints`/`getPipelineBlueprint` APIs.
88 */
89 pipelineConfigurationBody: string;
90 /**
91 * CloudWatch Logs publishing configuration for pipeline logs.
92 */
93 logPublishingOptions?: PipelineLogPublishingOptions;
94 /**
95 * Persistent buffering for the pipeline's ingest data.
96 */
97 bufferOptions?: PipelineBufferOptions;
98 /**
99 * Customer-managed KMS encryption for buffer data.
100 * @default an AWS-owned key
101 */
102 encryptionAtRestOptions?: PipelineEncryptionAtRestOptions;
103 /**
104 * VPC placement for the pipeline's ingest endpoint. Omit for a public
105 * endpoint. Changing VPC options replaces the pipeline.
106 */
107 vpcOptions?: PipelineVpcOptions;
108 /**
109 * IAM role the pipeline assumes to write to its sinks (overrides the
110 * `sts_role_arn` inside the configuration body where supported).
111 */
112 pipelineRoleArn?: string;
113 /**
114 * User-defined tags for the pipeline.
115 */
116 tags?: Record<string, string>;
117 }
118
119 export interface Pipeline extends Resource<
120 "AWS.OSIS.Pipeline",
121 PipelineProps,
122 {
123 /**
124 * Name of the pipeline.
125 */
126 pipelineName: string;
127 /**
128 * ARN of the pipeline.
129 */
130 pipelineArn: string;
131 /**
132 * Pipeline status (e.g. `ACTIVE`, `CREATING`, `UPDATING`).
133 */
134 status: string;
135 /**
136 * Minimum Ingestion OCUs the pipeline scales down to.
137 */
138 minUnits: number | undefined;
139 /**
140 * Maximum Ingestion OCUs the pipeline scales up to.
141 */
142 maxUnits: number | undefined;
143 /**
144 * URLs to ingest data into the pipeline.
145 */
146 ingestEndpointUrls: string[] | undefined;
147 /**
148 * Tags on the pipeline.
149 */
150 tags: Record<string, string>;
151 },
152 never,
153 Providers
154 > {}
155
156 /**
157 * An Amazon OpenSearch Ingestion (OSIS) pipeline — a managed Data Prepper
158 * pipeline that ingests, transforms, and delivers data to OpenSearch domains,
159 * serverless collections, or S3.
160 *
161 * Pipelines take roughly 5-10 minutes to provision and are billed per
162 * Ingestion-OCU-hour while they exist (minimum 1 OCU). Destroy pipelines you
163 * are not using.
164 * ### Creating a Pipeline
165 * **Example:** HTTP Source to S3 Sink
166 * ```typescript
167 * const pipeline = yield* Pipeline("Logs", {
168 * minUnits: 1,
169 * maxUnits: 1,
170 * pipelineConfigurationBody: Output.interpolate`version: "2"
171 * log-pipeline:
172 * source:
173 * http:
174 * path: "/logs"
175 * sink:
176 * - s3:
177 * aws:
178 * sts_role_arn: "${role.roleArn}"
179 * region: "us-west-2"
180 * bucket: "${bucket.bucketName}"
181 * threshold:
182 * event_collect_timeout: "60s"
183 * codec:
184 * ndjson:
185 * `,
186 * });
187 * ```
188 *
189 * **Example:** Pipeline with CloudWatch Logging
190 * ```typescript
191 * const pipeline = yield* Pipeline("Logs", {
192 * minUnits: 1,
193 * maxUnits: 2,
194 * pipelineConfigurationBody: configYaml,
195 * logPublishingOptions: {
196 * isLoggingEnabled: true,
197 * cloudWatchLogDestination: {
198 * logGroup: "/aws/vendedlogs/OpenSearchIngestion/logs",
199 * },
200 * },
201 * });
202 * ```
203 *
204 * @resource
205 */
206 export const Pipeline = Resource<Pipeline>("AWS.OSIS.Pipeline");
207
208 const toLogPublishingOptions = (
209 options: PipelineLogPublishingOptions | undefined,
210 ): osis.LogPublishingOptions | undefined =>
211 options === undefined
212 ? undefined
213 : {
214 IsLoggingEnabled: options.isLoggingEnabled,
215 CloudWatchLogDestination:
216 options.cloudWatchLogDestination !== undefined
217 ? { LogGroup: options.cloudWatchLogDestination.logGroup }
218 : undefined,
219 };
220
221 export const PipelineProvider = () =>
222 Provider.effect(
223 Pipeline,
224 Effect.gen(function* () {
225 const toName = (id: string, props: Partial<PipelineProps>) =>
226 props.pipelineName
227 ? Effect.succeed(props.pipelineName)
228 : createPhysicalName({ id, maxLength: 28, lowercase: true });
229
230 const readPipeline = Effect.fn(function* (name: string) {
231 return yield* osis.getPipeline({ PipelineName: name }).pipe(
232 Effect.map((response) => response.Pipeline),
233 Effect.catchTag("ResourceNotFoundException", () => Effect.undefined),
234 );
235 });
236
237 const waitForActive = Effect.fn(function* (name: string) {
238 const pipeline = yield* waitForPipelineSettled(name, readPipeline(name));
239 if (pipeline === undefined) {
240 return yield* new OSISPipelineUnexpectedResponse({
241 message: `OSIS pipeline '${name}' not found while waiting`,
242 });
243 }
244 return pipeline;
245 });
246
247 const toAttrs = Effect.fn(function* (pipeline: osis.Pipeline) {
248 if (!pipeline.PipelineName || !pipeline.PipelineArn) {
249 return yield* new OSISPipelineUnexpectedResponse({
250 message: `OSIS pipeline '${pipeline.PipelineName}' is missing its ARN`,
251 });
252 }
253 return {
254 pipelineName: pipeline.PipelineName,
255 pipelineArn: pipeline.PipelineArn,
256 status: pipeline.Status ?? "ACTIVE",
257 minUnits: pipeline.MinUnits,
258 maxUnits: pipeline.MaxUnits,
259 ingestEndpointUrls: pipeline.IngestEndpointUrls,
260 tags: yield* readPipelineTags(pipeline.PipelineArn),
261 };
262 });
263
264 return {
265 stables: ["pipelineName", "pipelineArn"],
266
267 diff: Effect.fn(function* ({ id, olds, news }) {
268 if (!isResolved(news)) return undefined;
269 const o: Partial<PipelineProps> = olds ?? {};
270 const n: Partial<PipelineProps> = news ?? {};
271 if ((yield* toName(id, o)) !== (yield* toName(id, n))) {
272 return { action: "replace" } as const;
273 }
274 // VPC options are create-only.
275 if (!jsonEquals(o.vpcOptions, n.vpcOptions)) {
276 return { action: "replace" } as const;
277 }
278 }),
279
280 read: Effect.fn(function* ({ id, olds, output }) {
281 const name = output?.pipelineName ?? (yield* toName(id, olds ?? {}));
282 const pipeline = yield* readPipeline(name);
283 if (pipeline === undefined) return undefined;
284 const attrs = yield* toAttrs(pipeline);
285 return (yield* hasAlchemyTags(id, attrs.tags)) ? attrs : Unowned(attrs);
286 }),
287
288 reconcile: Effect.fn(function* ({ id, news, output, session }) {
289 const props = news!;
290 const name = output?.pipelineName ?? (yield* toName(id, props));
291 const internalTags = yield* createInternalTags(id);
292 const desiredTags = { ...internalTags, ...props.tags };
293
294 // 1. Observe — cloud state is authoritative.
295 let observed = yield* readPipeline(name);
296
297 // 2. Ensure — create if missing; tolerate AlreadyExists as a race.
298 if (observed === undefined) {
299 yield* osis
300 .createPipeline({
301 PipelineName: name,
302 MinUnits: props.minUnits,
303 MaxUnits: props.maxUnits,
304 PipelineConfigurationBody: props.pipelineConfigurationBody,
305 LogPublishingOptions: toLogPublishingOptions(props.logPublishingOptions),
306 BufferOptions:
307 props.bufferOptions !== undefined
308 ? {
309 PersistentBufferEnabled: props.bufferOptions.persistentBufferEnabled,
310 }
311 : undefined,
312 EncryptionAtRestOptions:
313 props.encryptionAtRestOptions !== undefined
314 ? { KmsKeyArn: props.encryptionAtRestOptions.kmsKeyArn }
315 : undefined,
316 VpcOptions:
317 props.vpcOptions !== undefined
318 ? {
319 SubnetIds: props.vpcOptions.subnetIds,
320 SecurityGroupIds: props.vpcOptions.securityGroupIds,
321 }
322 : undefined,
323 PipelineRoleArn: props.pipelineRoleArn,
324 Tags: Object.entries(desiredTags).map(([Key, Value]) => ({
325 Key,
326 Value,
327 })),
328 })
329 .pipe(Effect.catchTag("ResourceAlreadyExistsException", () => Effect.void));
330 }
331
332 // Creation and in-flight updates both surface as transitional
333 // statuses; wait (bounded) so update calls do not conflict.
334 observed = yield* waitForActive(name);
335
336 // 3. Sync — compute the update delta from OBSERVED state.
337 const update: osis.UpdatePipelineRequest = { PipelineName: name };
338 let mutated = false;
339 if (props.minUnits !== observed.MinUnits) {
340 update.MinUnits = props.minUnits;
341 mutated = true;
342 }
343 if (props.maxUnits !== observed.MaxUnits) {
344 update.MaxUnits = props.maxUnits;
345 mutated = true;
346 }
347 if (
348 props.pipelineConfigurationBody.trim() !==
349 (observed.PipelineConfigurationBody ?? "").trim()
350 ) {
351 update.PipelineConfigurationBody = props.pipelineConfigurationBody;
352 mutated = true;
353 }
354 const desiredLogging = toLogPublishingOptions(props.logPublishingOptions);
355 if (
356 desiredLogging !== undefined &&
357 !jsonEquals(
358 {
359 IsLoggingEnabled: observed.LogPublishingOptions?.IsLoggingEnabled,
360 CloudWatchLogDestination: observed.LogPublishingOptions?.CloudWatchLogDestination,
361 },
362 desiredLogging,
363 )
364 ) {
365 update.LogPublishingOptions = desiredLogging;
366 mutated = true;
367 }
368 if (
369 props.bufferOptions !== undefined &&
370 props.bufferOptions.persistentBufferEnabled !==
371 observed.BufferOptions?.PersistentBufferEnabled
372 ) {
373 update.BufferOptions = {
374 PersistentBufferEnabled: props.bufferOptions.persistentBufferEnabled,
375 };
376 mutated = true;
377 }
378 if (
379 props.encryptionAtRestOptions !== undefined &&
380 props.encryptionAtRestOptions.kmsKeyArn !== observed.EncryptionAtRestOptions?.KmsKeyArn
381 ) {
382 update.EncryptionAtRestOptions = {
383 KmsKeyArn: props.encryptionAtRestOptions.kmsKeyArn,
384 };
385 mutated = true;
386 }
387 if (
388 props.pipelineRoleArn !== undefined &&
389 props.pipelineRoleArn !== observed.PipelineRoleArn
390 ) {
391 update.PipelineRoleArn = props.pipelineRoleArn;
392 mutated = true;
393 }
394
395 if (mutated) {
396 yield* osis.updatePipeline(update);
397 observed = yield* waitForActive(name);
398 }
399
400 // 3b. Sync tags — diff against OBSERVED cloud tags.
401 const arn = observed.PipelineArn;
402 if (arn) {
403 const observedTags = yield* readPipelineTags(arn);
404 const { removed, upsert } = diffTags(observedTags, desiredTags);
405 if (upsert.length > 0) {
406 yield* osis.tagResource({ Arn: arn, Tags: upsert });
407 }
408 if (removed.length > 0) {
409 yield* osis.untagResource({ Arn: arn, TagKeys: removed });
410 }
411 }
412
413 yield* session.note(name);
414 return yield* toAttrs(observed);
415 }),
416
417 delete: Effect.fn(function* ({ output }) {
418 const name = output.pipelineName;
419 // A pipeline mid-create/update rejects deletion with
420 // ConflictException — wait (bounded, tolerant) for it to settle,
421 // then retry conflicts through any residual transition.
422 yield* waitForPipelineSettled(name, readPipeline(name)).pipe(
423 Effect.orElseSucceed(() => undefined),
424 );
425 yield* retryWhilePipelineConflict(
426 osis
427 .deletePipeline({ PipelineName: name })
428 .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void)),
429 );
430 }),
431
432 list: () =>
433 osis.listPipelines.pages({}).pipe(
434 Stream.runCollect,
435 Effect.map((chunk) =>
436 Array.from(chunk).flatMap((page) =>
437 (page.Pipelines ?? [])
438 .map((summary) => summary.PipelineName)
439 .filter((name): name is string => name !== undefined),
440 ),
441 ),
442 Effect.flatMap(
443 Effect.forEach(
444 (name) =>
445 osis.getPipeline({ PipelineName: name }).pipe(
446 Effect.map((response) => response.Pipeline),
447 Effect.catchTag("ResourceNotFoundException", () => Effect.undefined),
448 ),
449 { concurrency: 4 },
450 ),
451 ),
452 Effect.map((pipelines) =>
453 pipelines.filter((pipeline): pipeline is osis.Pipeline => pipeline !== undefined),
454 ),
455 Effect.flatMap(
456 Effect.forEach((pipeline) => toAttrs(pipeline), {
457 concurrency: 4,
458 }),
459 ),
460 ),
461 };
462 }),
463 );