@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 455 lines 19.4 KB
Raw
1 import * as AWS from "@alchemy.run/aws";
2 import { describe, expect } from "@alchemy.run/test";
3 import * as Test from "@alchemy.run/test/Alchemy";
4 import * as iam from "@distilled.cloud/aws/iam";
5 import * as lf from "@distilled.cloud/aws/lakeformation";
6 import * as sts from "@distilled.cloud/aws/sts";
7 import * as Effect from "effect/Effect";
8 import * as Stream from "effect/Stream";
9 import { driftOf, repairDrift, waitForDrift } from "../drift.ts";
10
11 const { test } = Test.make({ providers: AWS.providers() });
12
13 /**
14 * LF-tag operations and permission grants require the caller to be a data
15 * lake administrator. Each stack bootstraps the caller as admin via
16 * `DataLakeSettings` (managed additively + restored on destroy). Derive the
17 * caller's IAM role ARN (with path — `GetRole` returns it, the assumed-role
18 * STS ARN does not carry the path).
19 */
20 const callerPrincipalArn = Effect.gen(function* () {
21 const identity = yield* sts.getCallerIdentity({});
22 const arn = identity.Arn!;
23 const match = /assumed-role\/([^/]+)\//.exec(arn);
24 if (match === null) return arn;
25 const role = yield* iam.getRole({ RoleName: match[1]! });
26 return role.Role.Arn;
27 });
28
29 const adminIds = Effect.gen(function* () {
30 const settings = (yield* lf.getDataLakeSettings({})).DataLakeSettings ?? {};
31 return (settings.DataLakeAdmins ?? [])
32 .map((p) => p.DataLakePrincipalIdentifier)
33 .filter((id): id is string => id !== undefined);
34 });
35
36 /** Raw (unfiltered) permission entries a principal holds on a database. */
37 const principalDatabasePermissions = (principal: string, databaseName: string) =>
38 Effect.gen(function* () {
39 const pages = yield* lf.listPermissions
40 .pages({ Resource: { Database: { Name: databaseName } } })
41 .pipe(Stream.runCollect);
42 return Array.from(pages)
43 .flatMap((p) => p.PrincipalResourcePermissions ?? [])
44 .filter((e) => e.Principal?.DataLakePrincipalIdentifier === principal)
45 .flatMap((e) => e.Permissions ?? [])
46 .sort();
47 });
48
49 const trustPolicy: AWS.IAM.PolicyDocument = {
50 Version: "2012-10-17",
51 Statement: [
52 { Effect: "Allow", Principal: { Service: "glue.amazonaws.com" }, Action: ["sts:AssumeRole"] },
53 ],
54 };
55
56 // All three tests mutate the account-level DataLakeSettings singleton
57 // (adding the caller / a scratch role as admin) — run them sequentially so
58 // concurrent read-modify-write cycles cannot clobber each other.
59 describe.sequential(
60 "LakeFormation",
61 { tags: ["provider:aws", "provider:aws:iam", "provider:aws:lakeformation", "live"] },
62 () => {
63 test.provider(
64 "DataLakeSettings adds admins additively and restores on destroy",
65 (stack) =>
66 Effect.gen(function* () {
67 yield* stack.destroy();
68 const beforeAdmins = yield* adminIds;
69
70 const created = yield* stack.deploy(
71 Effect.gen(function* () {
72 const role = yield* AWS.IAM.Role("LfAdminRole", {
73 assumeRolePolicyDocument: trustPolicy,
74 });
75 const settings = yield* AWS.LakeFormation.DataLakeSettings("Settings", {
76 dataLakeAdmins: [role.roleArn],
77 });
78 return { role, settings };
79 }),
80 );
81
82 expect(created.settings.catalogId).toBeDefined();
83 expect(created.settings.dataLakeAdmins).toContain(created.role.roleArn);
84 expect(created.settings.managedAdmins).toEqual([created.role.roleArn]);
85
86 // out-of-band: our admin was added, pre-existing admins survived
87 const duringAdmins = yield* adminIds;
88 expect(duringAdmins).toContain(created.role.roleArn);
89 for (const admin of beforeAdmins) {
90 expect(duringAdmins).toContain(admin);
91 }
92
93 // destroy — only the admin we added is removed
94 yield* stack.destroy();
95 const afterAdmins = yield* adminIds;
96 expect(afterAdmins).not.toContain(created.role.roleArn);
97 for (const admin of beforeAdmins) {
98 expect(afterAdmins).toContain(admin);
99 }
100 }),
101 { timeout: 180_000 },
102 );
103
104 // #2688: `read` must observe the live value of each managed setting,
105 // not carry the captured ones forward.
106 test.provider(
107 "DataLakeSettings drift detects and repairs an out-of-band managed setting",
108 (stack) =>
109 Effect.gen(function* () {
110 yield* stack.destroy();
111 yield* stack.deploy(
112 AWS.LakeFormation.DataLakeSettings("DriftSettings", {
113 authorizedSessionTagValueList: ["alchemy-drift-declared"],
114 }),
115 );
116 expect((yield* driftOf(stack, "DriftSettings"))?.action).toBe("unchanged");
117
118 const current = (yield* lf.getDataLakeSettings({})).DataLakeSettings ?? {};
119 yield* lf.putDataLakeSettings({
120 DataLakeSettings: {
121 ...current,
122 AuthorizedSessionTagValueList: ["alchemy-drift-out-of-band"],
123 },
124 });
125 expect(
126 (yield* waitForDrift(stack, "DriftSettings", "drifted"))?.attr?.managedSettings
127 ?.authorizedSessionTagValueList,
128 ).toEqual(["alchemy-drift-out-of-band"]);
129
130 yield* repairDrift(stack);
131 expect((yield* driftOf(stack, "DriftSettings"))?.action).toBe("unchanged");
132 const restored = (yield* lf.getDataLakeSettings({})).DataLakeSettings ?? {};
133 expect(restored.AuthorizedSessionTagValueList).toEqual(["alchemy-drift-declared"]);
134
135 yield* stack.destroy();
136 }),
137 { timeout: 180_000 },
138 );
139
140 test.provider(
141 "Permissions grants, updates, and revokes on a Glue database",
142 (stack) =>
143 Effect.gen(function* () {
144 yield* stack.destroy();
145 const admin = yield* callerPrincipalArn;
146
147 const created = yield* stack.deploy(
148 Effect.gen(function* () {
149 const settings = yield* AWS.LakeFormation.DataLakeSettings("Admin", {
150 dataLakeAdmins: [admin],
151 });
152 const database = yield* AWS.Glue.Database("LfDb", {});
153 const role = yield* AWS.IAM.Role("LfAnalyst", {
154 assumeRolePolicyDocument: trustPolicy,
155 });
156 const grant = yield* AWS.LakeFormation.Permissions("AnalystDb", {
157 // consume a settings output so admin bootstrap deploys first
158 // and is destroyed last
159 catalogId: settings.catalogId,
160 principal: role.roleArn,
161 resource: { database: { name: database.databaseName } },
162 permissions: ["CREATE_TABLE", "DESCRIBE"],
163 });
164 return { settings, database, role, grant };
165 }),
166 );
167
168 expect(created.grant.permissions).toEqual(["CREATE_TABLE", "DESCRIBE"]);
169 expect(created.grant.resource.Database?.Name).toEqual(created.database.databaseName);
170
171 // out-of-band verification (raw entries, not principal-filtered)
172 const observed = yield* principalDatabasePermissions(
173 created.role.roleArn,
174 created.database.databaseName,
175 );
176 expect(observed).toEqual(["CREATE_TABLE", "DESCRIBE"]);
177
178 // update — swap CREATE_TABLE for ALTER (grant + revoke delta)
179 const updated = yield* stack.deploy(
180 Effect.gen(function* () {
181 const settings = yield* AWS.LakeFormation.DataLakeSettings("Admin", {
182 dataLakeAdmins: [admin],
183 });
184 const database = yield* AWS.Glue.Database("LfDb", {});
185 const role = yield* AWS.IAM.Role("LfAnalyst", {
186 assumeRolePolicyDocument: trustPolicy,
187 });
188 const grant = yield* AWS.LakeFormation.Permissions("AnalystDb", {
189 catalogId: settings.catalogId,
190 principal: role.roleArn,
191 resource: { database: { name: database.databaseName } },
192 permissions: ["ALTER", "DESCRIBE"],
193 });
194 return { settings, database, role, grant };
195 }),
196 );
197
198 expect(updated.grant.permissions).toEqual(["ALTER", "DESCRIBE"]);
199 const reobserved = yield* principalDatabasePermissions(
200 created.role.roleArn,
201 created.database.databaseName,
202 );
203 expect(reobserved).toEqual(["ALTER", "DESCRIBE"]);
204
205 // remove the grant from the stack — permissions are revoked while
206 // the database still exists, so we can verify out-of-band
207 yield* stack.deploy(
208 Effect.gen(function* () {
209 const settings = yield* AWS.LakeFormation.DataLakeSettings("Admin", {
210 dataLakeAdmins: [admin],
211 });
212 const database = yield* AWS.Glue.Database("LfDb", {});
213 const role = yield* AWS.IAM.Role("LfAnalyst", {
214 assumeRolePolicyDocument: trustPolicy,
215 });
216 return { settings, database, role };
217 }),
218 );
219
220 const revoked = yield* principalDatabasePermissions(
221 created.role.roleArn,
222 created.database.databaseName,
223 );
224 expect(revoked).toEqual([]);
225
226 yield* stack.destroy();
227 }),
228 { tags: ["provider:aws:glue"], timeout: 240_000 },
229 );
230
231 test.provider(
232 "LFTag lifecycle and LFTagAssociation on a Glue database",
233 (stack) =>
234 Effect.gen(function* () {
235 yield* stack.destroy();
236 const admin = yield* callerPrincipalArn;
237
238 const created = yield* stack.deploy(
239 Effect.gen(function* () {
240 const settings = yield* AWS.LakeFormation.DataLakeSettings("Admin", {
241 dataLakeAdmins: [admin],
242 });
243 const tag = yield* AWS.LakeFormation.LFTag("EnvTag", {
244 catalogId: settings.catalogId,
245 tagKey: "alchemy-lf-env",
246 tagValues: ["dev", "prod"],
247 });
248 const database = yield* AWS.Glue.Database("LfTagDb", {});
249 const association = yield* AWS.LakeFormation.LFTagAssociation("DbEnv", {
250 catalogId: settings.catalogId,
251 resource: { database: { name: database.databaseName } },
252 lfTags: [{ tagKey: tag.tagKey, tagValues: ["dev"] }],
253 });
254 return { settings, tag, database, association };
255 }),
256 );
257
258 expect(created.tag.tagKey).toEqual("alchemy-lf-env");
259 expect([...created.tag.tagValues].sort()).toEqual(["dev", "prod"]);
260
261 // out-of-band: tag definition + assignment on the database
262 const observedTag = yield* lf.getLFTag({ TagKey: "alchemy-lf-env" });
263 expect([...(observedTag.TagValues ?? [])].sort()).toEqual(["dev", "prod"]);
264 const observedAssignment = yield* lf.getResourceLFTags({
265 Resource: { Database: { Name: created.database.databaseName } },
266 });
267 expect(
268 observedAssignment.LFTagOnDatabase?.find((t) => t.TagKey === "alchemy-lf-env")
269 ?.TagValues,
270 ).toEqual(["dev"]);
271
272 // update — add a tag value and move the assignment onto it
273 yield* stack.deploy(
274 Effect.gen(function* () {
275 const settings = yield* AWS.LakeFormation.DataLakeSettings("Admin", {
276 dataLakeAdmins: [admin],
277 });
278 const tag = yield* AWS.LakeFormation.LFTag("EnvTag", {
279 catalogId: settings.catalogId,
280 tagKey: "alchemy-lf-env",
281 tagValues: ["dev", "prod", "staging"],
282 });
283 const database = yield* AWS.Glue.Database("LfTagDb", {});
284 const association = yield* AWS.LakeFormation.LFTagAssociation("DbEnv", {
285 catalogId: settings.catalogId,
286 resource: { database: { name: database.databaseName } },
287 lfTags: [{ tagKey: tag.tagKey, tagValues: ["staging"] }],
288 });
289 return { settings, tag, database, association };
290 }),
291 );
292
293 const updatedTag = yield* lf.getLFTag({ TagKey: "alchemy-lf-env" });
294 expect([...(updatedTag.TagValues ?? [])].sort()).toEqual(["dev", "prod", "staging"]);
295 const updatedAssignment = yield* lf.getResourceLFTags({
296 Resource: { Database: { Name: created.database.databaseName } },
297 });
298 expect(
299 updatedAssignment.LFTagOnDatabase?.find((t) => t.TagKey === "alchemy-lf-env")
300 ?.TagValues,
301 ).toEqual(["staging"]);
302
303 // remove tag + association from the stack (settings stay, so the
304 // caller is still admin and can verify the deletion out-of-band —
305 // GetLFTag as a non-admin is AccessDenied, not EntityNotFound)
306 yield* stack.deploy(
307 Effect.gen(function* () {
308 const settings = yield* AWS.LakeFormation.DataLakeSettings("Admin", {
309 dataLakeAdmins: [admin],
310 });
311 return { settings };
312 }),
313 );
314 const goneTag = yield* lf
315 .getLFTag({ TagKey: "alchemy-lf-env" })
316 .pipe(Effect.catchTag("EntityNotFoundException", () => Effect.succeed(undefined)));
317 expect(goneTag).toBeUndefined();
318
319 yield* stack.destroy();
320 }),
321 { tags: ["provider:aws:glue"], timeout: 240_000 },
322 );
323
324 test.provider(
325 "LFTagExpression, DataCellsFilter, and OptIn lifecycle",
326 (stack) =>
327 Effect.gen(function* () {
328 yield* stack.destroy();
329 const admin = yield* callerPrincipalArn;
330
331 const build = (options: { expressionValues: string[]; rowFilterExpression?: string }) =>
332 Effect.gen(function* () {
333 const settings = yield* AWS.LakeFormation.DataLakeSettings("Admin", {
334 dataLakeAdmins: [admin],
335 });
336 const tag = yield* AWS.LakeFormation.LFTag("ExprTag", {
337 catalogId: settings.catalogId,
338 tagKey: "alchemy-lf-expr",
339 tagValues: ["a", "b"],
340 });
341 const expression = yield* AWS.LakeFormation.LFTagExpression("Expr", {
342 catalogId: settings.catalogId,
343 name: "alchemy-lf-expression",
344 description: "alchemy test expression",
345 expression: [{ tagKey: tag.tagKey, tagValues: options.expressionValues }],
346 });
347 const database = yield* AWS.Glue.Database("FilterDb", {});
348 const table = yield* AWS.Glue.Table("FilterTable", {
349 databaseName: database.databaseName,
350 tableType: "EXTERNAL_TABLE",
351 storageDescriptor: {
352 location: "s3://example-bucket/lf-filter/",
353 inputFormat: "org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat",
354 outputFormat: "org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat",
355 serdeInfo: {
356 serializationLibrary:
357 "org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe",
358 },
359 columns: [
360 { name: "id", type: "string" },
361 { name: "email", type: "string" },
362 ],
363 },
364 });
365 const filter = yield* AWS.LakeFormation.DataCellsFilter("NoEmail", {
366 tableCatalogId: settings.catalogId,
367 databaseName: database.databaseName,
368 tableName: table.tableName,
369 name: "alchemy-no-email",
370 excludedColumnNames: ["email"],
371 rowFilter:
372 options.rowFilterExpression !== undefined
373 ? { filterExpression: options.rowFilterExpression }
374 : undefined,
375 });
376 const role = yield* AWS.IAM.Role("OptInRole", {
377 assumeRolePolicyDocument: trustPolicy,
378 });
379 const optIn = yield* AWS.LakeFormation.OptIn("DbOptIn", {
380 principal: role.roleArn,
381 resource: {
382 // consume settings.catalogId so admin bootstrap deploys first
383 database: { catalogId: settings.catalogId, name: database.databaseName },
384 },
385 });
386 return { settings, tag, expression, database, table, filter, role, optIn };
387 });
388
389 const created = yield* stack.deploy(build({ expressionValues: ["a"] }));
390
391 expect(created.expression.name).toEqual("alchemy-lf-expression");
392 expect(created.expression.expression).toEqual([
393 { tagKey: "alchemy-lf-expr", tagValues: ["a"] },
394 ]);
395 expect(created.filter.name).toEqual("alchemy-no-email");
396 expect(created.optIn.principal).toEqual(created.role.roleArn);
397
398 // out-of-band verification
399 const observedExpr = yield* lf.getLFTagExpression({ Name: "alchemy-lf-expression" });
400 expect(observedExpr.Expression?.[0]?.TagKey).toEqual("alchemy-lf-expr");
401 expect(observedExpr.Expression?.[0]?.TagValues).toEqual(["a"]);
402 const observedFilter = yield* lf.getDataCellsFilter({
403 TableCatalogId: created.filter.tableCatalogId,
404 DatabaseName: created.database.databaseName,
405 TableName: created.table.tableName,
406 Name: "alchemy-no-email",
407 });
408 expect(observedFilter.DataCellsFilter?.ColumnWildcard?.ExcludedColumnNames).toEqual([
409 "email",
410 ]);
411 const observedOptIns = yield* lf.listLakeFormationOptIns({
412 Principal: { DataLakePrincipalIdentifier: created.role.roleArn },
413 Resource: { Database: { Name: created.database.databaseName } },
414 });
415 expect(observedOptIns.LakeFormationOptInsInfoList?.length).toBeGreaterThanOrEqual(1);
416
417 // update — swap the expression's tag values and add a row filter
418 yield* stack.deploy(build({ expressionValues: ["b"], rowFilterExpression: "id='x'" }));
419
420 const updatedExpr = yield* lf.getLFTagExpression({ Name: "alchemy-lf-expression" });
421 expect(updatedExpr.Expression?.[0]?.TagValues).toEqual(["b"]);
422 const updatedFilter = yield* lf.getDataCellsFilter({
423 TableCatalogId: created.filter.tableCatalogId,
424 DatabaseName: created.database.databaseName,
425 TableName: created.table.tableName,
426 Name: "alchemy-no-email",
427 });
428 expect(updatedFilter.DataCellsFilter?.RowFilter?.FilterExpression).toEqual("id='x'");
429
430 // remove everything but the admin bootstrap so deletion can be
431 // verified out-of-band while the caller is still an admin
432 yield* stack.deploy(
433 Effect.gen(function* () {
434 const settings = yield* AWS.LakeFormation.DataLakeSettings("Admin", {
435 dataLakeAdmins: [admin],
436 });
437 return { settings };
438 }),
439 );
440
441 const goneExpr = yield* lf
442 .getLFTagExpression({ Name: "alchemy-lf-expression" })
443 .pipe(Effect.catchTag("EntityNotFoundException", () => Effect.succeed(undefined)));
444 expect(goneExpr).toBeUndefined();
445 const goneOptIns = yield* lf.listLakeFormationOptIns({
446 Principal: { DataLakePrincipalIdentifier: created.role.roleArn },
447 });
448 expect(goneOptIns.LakeFormationOptInsInfoList ?? []).toHaveLength(0);
449
450 yield* stack.destroy();
451 }),
452 { tags: ["provider:aws:glue"], timeout: 240_000 },
453 );
454 },
455 );