The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import * as securitylake from "@distilled.cloud/aws/securitylake"; |
| 2 | import { Unowned } from "alchemy/AdoptPolicy"; |
| 3 | import * as Provider from "alchemy/Provider"; |
| 4 | import { Resource } from "alchemy/Resource"; |
| 5 | import { createInternalTags, diffTags, hasAlchemyTags } from "alchemy/Tags"; |
| 6 | import { toWireDays } from "alchemy/Util/Duration"; |
| 7 | import * as Data from "effect/Data"; |
| 8 | import type * as Duration from "effect/Duration"; |
| 9 | import * as Effect from "effect/Effect"; |
| 10 | import { AWSEnvironment } from "../Environment.ts"; |
| 11 | import type { Providers } from "../Providers.ts"; |
| 12 | import { readSecurityLakeTags, retryWhileConflict, toTagList } from "./internal.ts"; |
| 13 | |
| 14 | /** |
| 15 | * Encryption settings for a Security Lake Region. |
| 16 | */ |
| 17 | export interface DataLakeEncryptionConfiguration { |
| 18 | /** |
| 19 | * The KMS key ID used to encrypt objects at rest, or `S3_MANAGED_KEY` for |
| 20 | * SSE-S3 (the default). |
| 21 | * @default "S3_MANAGED_KEY" |
| 22 | */ |
| 23 | kmsKeyId?: string; |
| 24 | } |
| 25 | |
| 26 | /** |
| 27 | * When objects expire out of the data lake. |
| 28 | */ |
| 29 | export interface DataLakeLifecycleExpiration { |
| 30 | /** |
| 31 | * How long after creation objects are deleted. Accepts any |
| 32 | * `Duration.Input` (e.g. `"365 days"`, `Duration.days(365)`; a bare |
| 33 | * number is milliseconds); the wire unit is whole days. |
| 34 | */ |
| 35 | days?: Duration.Input; |
| 36 | } |
| 37 | |
| 38 | /** |
| 39 | * A storage-class transition rule. |
| 40 | */ |
| 41 | export interface DataLakeLifecycleTransition { |
| 42 | /** The S3 storage class to transition into (e.g. `GLACIER`, `ONEZONE_IA`). */ |
| 43 | storageClass?: string; |
| 44 | /** |
| 45 | * How long after creation objects transition. Accepts any |
| 46 | * `Duration.Input` (e.g. `"30 days"`, `Duration.days(30)`; a bare |
| 47 | * number is milliseconds); the wire unit is whole days. |
| 48 | */ |
| 49 | days?: Duration.Input; |
| 50 | } |
| 51 | |
| 52 | /** |
| 53 | * Lifecycle management (retention + storage-class transitions) for a Region. |
| 54 | */ |
| 55 | export interface DataLakeLifecycleConfiguration { |
| 56 | /** When objects expire out of the data lake. */ |
| 57 | expiration?: DataLakeLifecycleExpiration; |
| 58 | /** Storage-class transition rules. */ |
| 59 | transitions?: DataLakeLifecycleTransition[]; |
| 60 | } |
| 61 | |
| 62 | /** |
| 63 | * Rollup-Region replication for a contributing Region. |
| 64 | */ |
| 65 | export interface DataLakeReplicationConfiguration { |
| 66 | /** The rollup Regions that this Region's data is replicated into. */ |
| 67 | regions?: string[]; |
| 68 | /** IAM role ARN Security Lake assumes to replicate objects. */ |
| 69 | roleArn?: string; |
| 70 | } |
| 71 | |
| 72 | /** |
| 73 | * Per-Region configuration of the Security Lake data lake. |
| 74 | */ |
| 75 | export interface DataLakeRegionConfiguration { |
| 76 | /** The Region in which Security Lake is enabled. */ |
| 77 | region: string; |
| 78 | /** Encryption at rest for this Region. */ |
| 79 | encryptionConfiguration?: DataLakeEncryptionConfiguration; |
| 80 | /** Retention and storage-class transitions for this Region. */ |
| 81 | lifecycleConfiguration?: DataLakeLifecycleConfiguration; |
| 82 | /** Rollup-Region replication for this Region. */ |
| 83 | replicationConfiguration?: DataLakeReplicationConfiguration; |
| 84 | } |
| 85 | |
| 86 | export interface DataLakeProps { |
| 87 | /** |
| 88 | * The Regions to enable Security Lake in, each with its own encryption, |
| 89 | * lifecycle, and replication settings. Adding a Region enables Security Lake |
| 90 | * there; removing a previously configured Region disables it there. |
| 91 | */ |
| 92 | configurations: DataLakeRegionConfiguration[]; |
| 93 | |
| 94 | /** |
| 95 | * ARN of the IAM role used by the Security Lake metastore manager (the |
| 96 | * `AmazonSecurityLakeMetaStoreManagerV2` role) to populate the Glue |
| 97 | * metastore with partition updates. |
| 98 | */ |
| 99 | metaStoreManagerRoleArn: string; |
| 100 | |
| 101 | /** |
| 102 | * Tags applied to the data lake. Alchemy ownership tags are merged in |
| 103 | * automatically so the data lake can be recognized on subsequent runs. |
| 104 | */ |
| 105 | tags?: Record<string, string>; |
| 106 | } |
| 107 | |
| 108 | /** Attributes of the data lake in one Region. */ |
| 109 | export interface DataLakeRegionAttributes { |
| 110 | /** ARN of the data lake in this Region. */ |
| 111 | dataLakeArn: string; |
| 112 | /** The Region. */ |
| 113 | region: string; |
| 114 | /** ARN of the S3 bucket Security Lake created for this Region. */ |
| 115 | s3BucketArn: string | undefined; |
| 116 | /** Creation status of the data lake in this Region. */ |
| 117 | createStatus: string | undefined; |
| 118 | } |
| 119 | |
| 120 | /** @resource */ |
| 121 | export interface DataLake extends Resource< |
| 122 | "AWS.SecurityLake.DataLake", |
| 123 | DataLakeProps, |
| 124 | { |
| 125 | /** ARN of the data lake in the current (deployment) Region. */ |
| 126 | dataLakeArn: string; |
| 127 | /** Every Region the data lake is enabled in. */ |
| 128 | regions: string[]; |
| 129 | /** Per-Region attributes (ARN, S3 bucket, status). */ |
| 130 | dataLakes: DataLakeRegionAttributes[]; |
| 131 | }, |
| 132 | never, |
| 133 | Providers |
| 134 | > {} |
| 135 | |
| 136 | /** |
| 137 | * The Amazon Security Lake data lake — the account-wide singleton that |
| 138 | * onboards the account to Security Lake. Enabling it creates S3 buckets, |
| 139 | * registers them with Lake Formation, and configures the Glue metastore in |
| 140 | * every configured Region. |
| 141 | * |
| 142 | * This is a heavyweight, account-wide resource: enabling/disabling Security |
| 143 | * Lake affects the whole account, and the S3 buckets it creates are retained |
| 144 | * after the data lake is deleted. |
| 145 | * |
| 146 | * ### Enabling Security Lake |
| 147 | * **Example:** Single-Region data lake |
| 148 | * ```typescript |
| 149 | * const lake = yield* SecurityLake.DataLake("Lake", { |
| 150 | * configurations: [{ region: "us-west-2" }], |
| 151 | * metaStoreManagerRoleArn: metastoreRole.roleArn, |
| 152 | * }); |
| 153 | * ``` |
| 154 | * |
| 155 | * **Example:** Lifecycle management and KMS encryption |
| 156 | * ```typescript |
| 157 | * const lake = yield* SecurityLake.DataLake("Lake", { |
| 158 | * configurations: [ |
| 159 | * { |
| 160 | * region: "us-west-2", |
| 161 | * encryptionConfiguration: { kmsKeyId: key.keyId }, |
| 162 | * lifecycleConfiguration: { |
| 163 | * expiration: { days: "365 days" }, |
| 164 | * transitions: [{ storageClass: "ONEZONE_IA", days: "30 days" }], |
| 165 | * }, |
| 166 | * }, |
| 167 | * ], |
| 168 | * metaStoreManagerRoleArn: metastoreRole.roleArn, |
| 169 | * tags: { team: "security" }, |
| 170 | * }); |
| 171 | * ``` |
| 172 | */ |
| 173 | const DataLakeResource = Resource<DataLake>("AWS.SecurityLake.DataLake"); |
| 174 | |
| 175 | export { DataLakeResource as DataLake }; |
| 176 | |
| 177 | /** |
| 178 | * The data lake failed to reach `COMPLETED` in one of the configured Regions. |
| 179 | */ |
| 180 | export class DataLakeCreateFailed extends Data.TaggedError("DataLakeCreateFailed")<{ |
| 181 | readonly region: string; |
| 182 | readonly reason: string | undefined; |
| 183 | readonly code: string | undefined; |
| 184 | }> {} |
| 185 | |
| 186 | // Convert a per-Region prop configuration (Duration-typed lifecycle days) |
| 187 | // into the wire shape the Security Lake API expects (whole days). |
| 188 | const toWireConfiguration = ( |
| 189 | config: DataLakeRegionConfiguration, |
| 190 | ): securitylake.DataLakeConfiguration => ({ |
| 191 | region: config.region, |
| 192 | encryptionConfiguration: config.encryptionConfiguration, |
| 193 | lifecycleConfiguration: config.lifecycleConfiguration |
| 194 | ? { |
| 195 | expiration: config.lifecycleConfiguration.expiration |
| 196 | ? { days: toWireDays(config.lifecycleConfiguration.expiration.days) } |
| 197 | : undefined, |
| 198 | transitions: config.lifecycleConfiguration.transitions?.map((transition) => ({ |
| 199 | storageClass: transition.storageClass, |
| 200 | days: toWireDays(transition.days), |
| 201 | })), |
| 202 | } |
| 203 | : undefined, |
| 204 | replicationConfiguration: config.replicationConfiguration, |
| 205 | }); |
| 206 | |
| 207 | // Normalize a per-Region config for observed↔desired comparison. AWS reports |
| 208 | // SSE-S3 as `S3_MANAGED_KEY`, so an unset kmsKeyId compares equal to it. |
| 209 | const normalizeConfig = (config: { |
| 210 | encryptionConfiguration?: securitylake.DataLakeEncryptionConfiguration; |
| 211 | lifecycleConfiguration?: securitylake.DataLakeLifecycleConfiguration; |
| 212 | replicationConfiguration?: securitylake.DataLakeReplicationConfiguration; |
| 213 | }) => |
| 214 | JSON.stringify({ |
| 215 | kmsKeyId: config.encryptionConfiguration?.kmsKeyId ?? "S3_MANAGED_KEY", |
| 216 | expirationDays: config.lifecycleConfiguration?.expiration?.days, |
| 217 | transitions: [...(config.lifecycleConfiguration?.transitions ?? [])] |
| 218 | .map((t) => ({ storageClass: t.storageClass, days: t.days })) |
| 219 | .sort((l, r) => (l.storageClass ?? "").localeCompare(r.storageClass ?? "")), |
| 220 | replicationRegions: [...(config.replicationConfiguration?.regions ?? [])].sort(), |
| 221 | replicationRoleArn: config.replicationConfiguration?.roleArn, |
| 222 | }); |
| 223 | |
| 224 | const buildRegionAttrs = (lake: securitylake.DataLakeResource): DataLakeRegionAttributes => ({ |
| 225 | dataLakeArn: lake.dataLakeArn, |
| 226 | region: lake.region, |
| 227 | s3BucketArn: lake.s3BucketArn, |
| 228 | createStatus: lake.createStatus, |
| 229 | }); |
| 230 | |
| 231 | const buildAttrs = (lakes: securitylake.DataLakeResource[], currentRegion: string) => { |
| 232 | const sorted = [...lakes].sort((l, r) => l.region.localeCompare(r.region)); |
| 233 | const current = sorted.find((lake) => lake.region === currentRegion); |
| 234 | return { |
| 235 | dataLakeArn: (current ?? sorted[0]!).dataLakeArn, |
| 236 | regions: sorted.map((lake) => lake.region), |
| 237 | dataLakes: sorted.map(buildRegionAttrs), |
| 238 | }; |
| 239 | }; |
| 240 | |
| 241 | // All data lakes for the account across every Region. |
| 242 | const observeDataLakes = securitylake |
| 243 | .listDataLakes({}) |
| 244 | .pipe(Effect.map((response) => [...(response.dataLakes ?? [])])); |
| 245 | |
| 246 | // Observation for read/list: an account that never onboarded Security Lake |
| 247 | // rejects listDataLakes with AccessDeniedException — that means "no data |
| 248 | // lake", not a failure. |
| 249 | const observeDataLakesIfOnboarded = observeDataLakes.pipe( |
| 250 | Effect.catchTag(["AccessDeniedException", "UnauthorizedException"], () => |
| 251 | Effect.succeed([] as securitylake.DataLakeResource[]), |
| 252 | ), |
| 253 | ); |
| 254 | |
| 255 | export const DataLakeProvider = () => |
| 256 | Provider.effect( |
| 257 | DataLakeResource, |
| 258 | Effect.gen(function* () { |
| 259 | // Bounded wait (~90s) for every desired Region to finish onboarding. |
| 260 | // A FAILED createStatus is surfaced as a typed error; still-PENDING |
| 261 | // Regions after the window are returned as-is (the next reconcile |
| 262 | // converges) rather than hanging. |
| 263 | const awaitDataLakeReady = Effect.fn(function* (regions: string[]) { |
| 264 | let lakes: securitylake.DataLakeResource[] = []; |
| 265 | for (let attempt = 0; attempt < 18; attempt++) { |
| 266 | lakes = yield* observeDataLakes; |
| 267 | const failed = lakes.find( |
| 268 | (lake) => regions.includes(lake.region) && lake.createStatus === "FAILED", |
| 269 | ); |
| 270 | if (failed) { |
| 271 | return yield* new DataLakeCreateFailed({ |
| 272 | region: failed.region, |
| 273 | reason: failed.updateStatus?.exception?.reason, |
| 274 | code: failed.updateStatus?.exception?.code, |
| 275 | }); |
| 276 | } |
| 277 | const allReady = regions.every((region) => |
| 278 | lakes.some((lake) => lake.region === region && lake.createStatus === "COMPLETED"), |
| 279 | ); |
| 280 | if (allReady) return lakes; |
| 281 | yield* Effect.sleep("5 seconds"); |
| 282 | } |
| 283 | return lakes; |
| 284 | }); |
| 285 | |
| 286 | return { |
| 287 | read: Effect.fn(function* ({ id }) { |
| 288 | const { region } = yield* AWSEnvironment.current; |
| 289 | const lakes = yield* observeDataLakesIfOnboarded; |
| 290 | if (lakes.length === 0) return undefined; |
| 291 | const attrs = buildAttrs(lakes, region); |
| 292 | const tags = yield* readSecurityLakeTags(attrs.dataLakeArn); |
| 293 | return (yield* hasAlchemyTags(id, tags)) ? attrs : Unowned(attrs); |
| 294 | }), |
| 295 | |
| 296 | // The data lake is an account-wide singleton spanning Regions. |
| 297 | list: () => |
| 298 | Effect.gen(function* () { |
| 299 | const { region } = yield* AWSEnvironment.current; |
| 300 | const lakes = yield* observeDataLakesIfOnboarded; |
| 301 | return lakes.length > 0 ? [buildAttrs(lakes, region)] : []; |
| 302 | }), |
| 303 | |
| 304 | reconcile: Effect.fn(function* ({ id, news, output, session }) { |
| 305 | const { region } = yield* AWSEnvironment.current; |
| 306 | const internalTags = yield* createInternalTags(id); |
| 307 | const desiredTags = { ...news.tags, ...internalTags }; |
| 308 | const desiredConfigs = news.configurations.map(toWireConfiguration); |
| 309 | const desiredRegions = desiredConfigs.map((c) => c.region); |
| 310 | |
| 311 | // 1. OBSERVE — cloud state is authoritative. |
| 312 | let lakes = yield* observeDataLakes; |
| 313 | const observedByRegion = new Map(lakes.map((lake) => [lake.region, lake])); |
| 314 | |
| 315 | // 2. ENSURE — createDataLake is additive per Region, so enabling |
| 316 | // missing Regions covers both greenfield onboarding and Region |
| 317 | // expansion. A ConflictException is a race with a concurrent |
| 318 | // enable — fall through to observation. |
| 319 | const missing = desiredConfigs.filter((config) => !observedByRegion.has(config.region)); |
| 320 | if (missing.length > 0) { |
| 321 | yield* securitylake |
| 322 | .createDataLake({ |
| 323 | configurations: missing, |
| 324 | metaStoreManagerRoleArn: news.metaStoreManagerRoleArn, |
| 325 | tags: toTagList(desiredTags), |
| 326 | }) |
| 327 | .pipe(Effect.catchTag("ConflictException", () => Effect.succeed({}))); |
| 328 | } |
| 329 | |
| 330 | // 3. SYNC per-Region settings — observed ↔ desired delta only. |
| 331 | const changed = desiredConfigs.filter((config) => { |
| 332 | const observed = observedByRegion.get(config.region); |
| 333 | return observed !== undefined && normalizeConfig(config) !== normalizeConfig(observed); |
| 334 | }); |
| 335 | if (changed.length > 0) { |
| 336 | yield* securitylake.updateDataLake({ |
| 337 | configurations: changed, |
| 338 | metaStoreManagerRoleArn: news.metaStoreManagerRoleArn, |
| 339 | }); |
| 340 | } |
| 341 | |
| 342 | // 3b. SYNC Region removals — only Regions this resource previously |
| 343 | // managed (recorded in output) are disabled; foreign Regions that |
| 344 | // happen to be enabled are left alone. |
| 345 | const staleRegions = (output?.regions ?? []).filter( |
| 346 | (managed) => !desiredRegions.includes(managed) && observedByRegion.has(managed), |
| 347 | ); |
| 348 | if (staleRegions.length > 0) { |
| 349 | yield* securitylake.deleteDataLake({ regions: staleRegions }).pipe(retryWhileConflict); |
| 350 | } |
| 351 | |
| 352 | // 3c. Wait (bounded) for onboarding to complete in every Region. |
| 353 | lakes = yield* awaitDataLakeReady(desiredRegions); |
| 354 | if (lakes.length === 0) { |
| 355 | return yield* new DataLakeCreateFailed({ |
| 356 | region: desiredRegions[0] ?? region, |
| 357 | reason: "data lake not observable after create", |
| 358 | code: undefined, |
| 359 | }); |
| 360 | } |
| 361 | const managed = lakes.filter((lake) => desiredRegions.includes(lake.region)); |
| 362 | const attrs = buildAttrs(managed.length > 0 ? managed : lakes, region); |
| 363 | |
| 364 | // 3d. SYNC tags — diff against OBSERVED cloud tags. |
| 365 | const observedTags = yield* readSecurityLakeTags(attrs.dataLakeArn); |
| 366 | const { upsert, removed } = diffTags(observedTags, desiredTags); |
| 367 | if (upsert.length > 0) { |
| 368 | yield* securitylake.tagResource({ |
| 369 | resourceArn: attrs.dataLakeArn, |
| 370 | tags: upsert.map((t) => ({ key: t.Key, value: t.Value })), |
| 371 | }); |
| 372 | } |
| 373 | if (removed.length > 0) { |
| 374 | yield* securitylake.untagResource({ |
| 375 | resourceArn: attrs.dataLakeArn, |
| 376 | tagKeys: removed, |
| 377 | }); |
| 378 | } |
| 379 | |
| 380 | // 4. RETURN fresh attributes. |
| 381 | yield* session.note(attrs.dataLakeArn); |
| 382 | return attrs; |
| 383 | }), |
| 384 | |
| 385 | // Disabling Security Lake retains the S3 buckets it created (AWS |
| 386 | // behavior); only log collection and the lake configuration go away. |
| 387 | delete: Effect.fn(function* ({ output }) { |
| 388 | if (output.regions.length === 0) return; |
| 389 | yield* securitylake.deleteDataLake({ regions: output.regions }).pipe( |
| 390 | retryWhileConflict, |
| 391 | Effect.catchTag("ResourceNotFoundException", () => Effect.void), |
| 392 | ); |
| 393 | }), |
| 394 | }; |
| 395 | }), |
| 396 | ); |