@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 242 lines 9.42 KB
Raw
1 import * as logs from "@distilled.cloud/aws/cloudwatch-logs";
2 import * as glue from "@distilled.cloud/aws/glue";
3 import * as lf from "@distilled.cloud/aws/lakeformation";
4 import { diffTags } from "alchemy/Tags";
5 import * as Effect from "effect/Effect";
6 import * as Schedule from "effect/Schedule";
7 import * as Stream from "effect/Stream";
8
9 /** ARN partition — the testing account is on the standard AWS partition. */
10 const PARTITION = "aws";
11
12 /** `arn:aws:glue:{region}:{account}:catalog` */
13 export const catalogArn = (region: string, accountId: string) =>
14 `arn:${PARTITION}:glue:${region}:${accountId}:catalog`;
15
16 /** `arn:aws:glue:{region}:{account}:database/{name}` */
17 export const databaseArn = (region: string, accountId: string, databaseName: string) =>
18 `arn:${PARTITION}:glue:${region}:${accountId}:database/${databaseName}`;
19
20 /** `arn:aws:glue:{region}:{account}:table/{database}/{table}` */
21 export const tableArn = (
22 region: string,
23 accountId: string,
24 databaseName: string,
25 tableName: string,
26 ) => `arn:${PARTITION}:glue:${region}:${accountId}:table/${databaseName}/${tableName}`;
27
28 /** `arn:aws:glue:{region}:{account}:crawler/{name}` */
29 export const crawlerArn = (region: string, accountId: string, crawlerName: string) =>
30 `arn:${PARTITION}:glue:${region}:${accountId}:crawler/${crawlerName}`;
31
32 /** `arn:aws:glue:{region}:{account}:job/{name}` */
33 export const jobArn = (region: string, accountId: string, jobName: string) =>
34 `arn:${PARTITION}:glue:${region}:${accountId}:job/${jobName}`;
35
36 /** `arn:aws:glue:{region}:{account}:connection/{name}` */
37 export const connectionArn = (region: string, accountId: string, connectionName: string) =>
38 `arn:${PARTITION}:glue:${region}:${accountId}:connection/${connectionName}`;
39
40 /**
41 * Fetch the observed Glue tags for a resource ARN as a plain record. Glue's
42 * `GetTags` returns a `{ Tags: { key: value } }` map (not the array shape most
43 * AWS services use). Tolerate a missing/untaggable resource as `{}`.
44 */
45 export const fetchObservedTags = Effect.fn("AWS.Glue.fetchObservedTags")(function* (
46 resourceArn: string,
47 ) {
48 const response = yield* glue
49 .getTags({ ResourceArn: resourceArn })
50 .pipe(Effect.orElseSucceed(() => ({ Tags: undefined })));
51 const tags = response.Tags ?? {};
52 return Object.fromEntries(
53 Object.entries(tags).filter((entry): entry is [string, string] => entry[1] !== undefined),
54 );
55 });
56
57 /**
58 * Sync a Glue resource's tags: diff OBSERVED cloud tags against desired and
59 * apply the delta via `TagResource` (map of adds/updates) / `UntagResource`
60 * (list of removed keys).
61 */
62 export const syncTags = Effect.fn("AWS.Glue.syncTags")(function* (
63 resourceArn: string,
64 observed: Record<string, string>,
65 desired: Record<string, string>,
66 ) {
67 const { upsert, removed } = diffTags(observed, desired);
68 if (upsert.length > 0) {
69 yield* glue.tagResource({
70 ResourceArn: resourceArn,
71 TagsToAdd: Object.fromEntries(upsert.map((t) => [t.Key, t.Value])),
72 });
73 }
74 if (removed.length > 0) {
75 yield* glue.untagResource({
76 ResourceArn: resourceArn,
77 TagsToRemove: removed,
78 });
79 }
80 });
81
82 /**
83 * Bounded retry through `CrawlerRunningException` — a crawler cannot be
84 * updated or deleted while a crawl is in progress. Explicitly typed so the
85 * conditional `Retry.Return` type does not leak into the provider's
86 * declaration emit and widen `AWS.providers()` for downstream consumers.
87 */
88 export const retryWhileCrawlerRunning = <A, E extends { _tag: string }, R>(
89 self: Effect.Effect<A, E, R>,
90 ): Effect.Effect<A, E, R> =>
91 Effect.retry(self, {
92 while: (e) => e._tag === "CrawlerRunningException",
93 schedule: Schedule.max([Schedule.fixed("3 seconds"), Schedule.recurs(20)]),
94 });
95
96 /**
97 * Bounded retry through `ConcurrentModificationException` — Glue catalog
98 * mutations occasionally race under concurrent reconciles. Explicitly typed
99 * for the same declaration-emit reason as above.
100 */
101 export const retryWhileConcurrentModification = <A, E extends { _tag: string }, R>(
102 self: Effect.Effect<A, E, R>,
103 ): Effect.Effect<A, E, R> =>
104 Effect.retry(self, {
105 while: (e) => e._tag === "ConcurrentModificationException",
106 schedule: Schedule.max([Schedule.fixed("2 seconds"), Schedule.recurs(10)]),
107 });
108
109 /**
110 * Bounded retry through `GlueRoleNotAssumable` — a freshly-created IAM role is
111 * not yet assumable by the Glue service (IAM propagation), which surfaces as a
112 * message-discriminated `InvalidInputException` (patched into a typed tag).
113 * Explicitly typed for the same declaration-emit reason as above.
114 */
115 export const retryWhileRoleNotAssumable = <A, E extends { _tag: string }, R>(
116 self: Effect.Effect<A, E, R>,
117 ): Effect.Effect<A, E, R> =>
118 Effect.retry(self, {
119 while: (e) => e._tag === "GlueRoleNotAssumable",
120 schedule: Schedule.max([Schedule.fixed("5 seconds"), Schedule.recurs(12)]),
121 });
122
123 /**
124 * Glue validates crawler S3 targets through a service credential that can lag
125 * a newly-created bucket. Retry only the distilled synthetic tag for the
126 * observed InvalidAccessKeyId propagation failure.
127 */
128 export const retryWhileCrawlerTargetNotReady = <A, E extends { _tag: string }, R>(
129 self: Effect.Effect<A, E, R>,
130 ): Effect.Effect<A, E, R> =>
131 Effect.retry(self, {
132 while: (e) => e._tag === "GlueS3TargetNotReady",
133 schedule: Schedule.max([Schedule.fixed("5 seconds"), Schedule.recurs(10)]),
134 });
135
136 /** Bounded retry for transient states that prevent crawler deletion. */
137 export const retryCrawlerDelete = <A, E extends { _tag: string }, R>(
138 self: Effect.Effect<A, E, R>,
139 ): Effect.Effect<A, E, R> =>
140 Effect.retry(self, {
141 while: (e) =>
142 e._tag === "CrawlerRunningException" ||
143 e._tag === "SchedulerTransitioningException" ||
144 e._tag === "OperationTimeoutException",
145 schedule: Schedule.max([Schedule.fixed("3 seconds"), Schedule.recurs(10)]),
146 });
147
148 /** Glue free-form parameter/label maps arrive with `undefined` values erased. */
149 export const cleanMap = (
150 map: Record<string, string | undefined> | undefined,
151 ): Record<string, string> =>
152 Object.fromEntries(
153 Object.entries(map ?? {}).filter((entry): entry is [string, string] => entry[1] !== undefined),
154 );
155
156 /**
157 * Glue writes crawler and job-run logs into SHARED, service-managed log
158 * groups (`/aws-glue/crawlers`, `/aws-glue/python-jobs/output`, …) that it
159 * auto-creates on first use. The groups belong to the Glue service, not to
160 * any one crawler/job, so a provider must never delete them — it reaps only
161 * the streams it owns (named after the crawler, or prefixed by a run id).
162 *
163 * Best-effort and bounded: log cleanup is auxiliary to the already-completed
164 * resource delete, so a missing group/stream is success and a slow API only
165 * logs a warning.
166 */
167 export const reapGlueLogStreams = (
168 logGroupNames: readonly string[],
169 streamPrefixes: readonly string[],
170 options: { exact?: boolean } = {},
171 ) =>
172 Effect.forEach(
173 logGroupNames.flatMap((logGroupName) =>
174 streamPrefixes.map((prefix) => ({ logGroupName, prefix })),
175 ),
176 ({ logGroupName, prefix }) =>
177 logs.describeLogStreams.items({ logGroupName, logStreamNamePrefix: prefix }).pipe(
178 Stream.mapEffect(
179 (stream) =>
180 stream.logStreamName === undefined ||
181 (options.exact === true && stream.logStreamName !== prefix)
182 ? Effect.void
183 : logs.deleteLogStream({ logGroupName, logStreamName: stream.logStreamName }).pipe(
184 Effect.retry({
185 while: (e) =>
186 e._tag === "OperationAbortedException" ||
187 e._tag === "ServiceUnavailableException",
188 schedule: Schedule.exponential("250 millis"),
189 times: 6,
190 }),
191 Effect.catchTag("ResourceNotFoundException", () => Effect.void),
192 ),
193 { concurrency: 5 },
194 ),
195 Stream.runDrain,
196 Effect.catchTag("ResourceNotFoundException", () => Effect.void),
197 Effect.timeoutOrElse({
198 duration: "30 seconds",
199 orElse: () =>
200 Effect.logWarning(`Timed out reaping Glue log streams ${logGroupName}/${prefix}*`),
201 }),
202 ),
203 { concurrency: 5, discard: true },
204 );
205
206 /**
207 * Delete a Glue database, tolerating one that is already gone.
208 *
209 * A database registered under Lake Formation (e.g. one a DataZone
210 * environment provisioned) only grants DROP to the principals LF lists — IAM
211 * admin rights are not enough. A Lake Formation admin may re-open it to IAM
212 * access control, after which the drop succeeds. A caller that is not an LF
213 * admin gets the original denial back.
214 */
215 export const deleteGlueDatabase = Effect.fn("AWS.Glue.deleteGlueDatabase")(function* (
216 name: string,
217 catalogId: string | undefined,
218 ) {
219 const drop = glue.deleteDatabase({ Name: name, CatalogId: catalogId }).pipe(
220 retryWhileConcurrentModification,
221 Effect.catchTag("EntityNotFoundException", () => Effect.void),
222 );
223 yield* drop.pipe(
224 Effect.catchTag("InsufficientLakeFormationPermissions", (error) =>
225 lf
226 .grantPermissions({
227 CatalogId: catalogId,
228 Principal: { DataLakePrincipalIdentifier: "IAM_ALLOWED_PRINCIPALS" },
229 Resource: { Database: { CatalogId: catalogId, Name: name } },
230 Permissions: ["ALL"],
231 })
232 .pipe(
233 retryWhileConcurrentModification,
234 Effect.catchTags({
235 EntityNotFoundException: () => Effect.void,
236 AccessDeniedException: () => Effect.fail(error),
237 }),
238 Effect.andThen(drop),
239 ),
240 ),
241 );
242 });