The best way to control Claude Code, Codex and more from any computer or phone

main
ts 320 lines 12.1 KB
Raw
1 import * as logs from "@distilled.cloud/aws/cloudwatch-logs";
2 import * as deadline from "@distilled.cloud/aws/deadline";
3 import { diffTags } from "alchemy/Tags";
4 import * as Effect from "effect/Effect";
5 import * as Redacted from "effect/Redacted";
6 import * as Schedule from "effect/Schedule";
7 import * as EffectStream from "effect/Stream";
8 import { AWSEnvironment } from "../Environment.ts";
9
10 /**
11 * Unwrap distilled `SensitiveString` values (`string | Redacted<string>`)
12 * to a plain string for drift comparison.
13 */
14 export const asPlain = (
15 value: string | Redacted.Redacted<string> | undefined,
16 ): string | undefined =>
17 value === undefined ? undefined : Redacted.isRedacted(value) ? Redacted.value(value) : value;
18
19 /**
20 * Build a Deadline Cloud ARN factory bound to the ambient account/region.
21 * Paths look like `farm/{farmId}`, `farm/{farmId}/queue/{queueId}`,
22 * `monitor/{monitorId}`.
23 */
24 export const deadlineArnOf = Effect.gen(function* () {
25 const { accountId, region } = yield* AWSEnvironment.current;
26 return (path: string) => `arn:aws:deadline:${region}:${accountId}:${path}`;
27 });
28
29 /**
30 * Read the observed tags of a Deadline resource. A missing resource (or a
31 * resource type without tag support) reads as an empty tag set.
32 */
33 export const fetchDeadlineTags = Effect.fn(function* (arn: string) {
34 const response = yield* deadline
35 .listTagsForResource({ resourceArn: arn })
36 .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.undefined));
37 return Object.fromEntries(
38 Object.entries(response?.tags ?? {}).filter(
39 (entry): entry is [string, string] => entry[1] !== undefined,
40 ),
41 );
42 });
43
44 /**
45 * Converge a Deadline resource's tags to the desired set, diffing against
46 * the OBSERVED cloud tags (never olds/output).
47 */
48 export const syncDeadlineTags = Effect.fn(function* (arn: string, desired: Record<string, string>) {
49 const observed = yield* fetchDeadlineTags(arn);
50 const { removed, upsert } = diffTags(observed, desired);
51 if (removed.length > 0) {
52 yield* deadline.untagResource({ resourceArn: arn, tagKeys: removed });
53 }
54 if (upsert.length > 0) {
55 yield* deadline.tagResource({
56 resourceArn: arn,
57 tags: Object.fromEntries(upsert.map(({ Key, Value }) => [Key, Value])),
58 });
59 }
60 });
61
62 /**
63 * Deadline auto-creates CloudWatch log groups under
64 * `/aws/deadline/{farmId}/{queueId}` (queue job/session logs) and
65 * `/aws/deadline/{farmId}/{fleetId}/...` (fleet worker logs), and neither
66 * `deleteQueue` nor `deleteFarm` removes them — without this reap every
67 * deleted farm strands orphaned log groups. Idempotent: a group already
68 * gone (or never created) is not an error.
69 */
70 export const reapDeadlineLogGroups = Effect.fn(function* (prefix: string) {
71 const groups = yield* logs.describeLogGroups.pages({ logGroupNamePrefix: prefix }).pipe(
72 EffectStream.runCollect,
73 Effect.map((chunk) =>
74 Array.from(chunk)
75 .flatMap((page) => page.logGroups ?? [])
76 .map((group) => group.logGroupName)
77 .filter((name): name is string => name != null),
78 ),
79 );
80 yield* Effect.forEach(
81 groups,
82 (logGroupName) =>
83 logs
84 .deleteLogGroup({ logGroupName })
85 .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void)),
86 { concurrency: 4, discard: true },
87 );
88 });
89
90 /**
91 * Reap every child resource of a farm so `deleteFarm` can succeed. A normal
92 * stack destroy deletes children before the farm, so this observes nothing —
93 * but an orphan sweep (nuke) or a mid-run crash can leave a farm whose
94 * children were never enumerated, and `deleteFarm` then rejects with
95 * `ConflictException` ("This farm still contains some storage profiles /
96 * queues / fleets ...") until the retry budget runs out and the farm leaks.
97 *
98 * Order matters: queue-fleet/queue-limit associations must be stopped and
99 * deleted before their queues/fleets/limits; storage profiles may be
100 * referenced by queues (`allowedStorageProfileIds`) so they go after the
101 * queues. Every step is idempotent — a child already gone is not an error.
102 */
103 export const reapFarmChildren = Effect.fn(function* (farmId: string) {
104 // Every list catches ResourceNotFoundException → [] because the farm
105 // itself may already be gone.
106
107 // 1. Queue-fleet associations — stop scheduling, then delete (the delete
108 // rejects with ConflictException until the stop settles to STOPPED).
109 const queueFleetAssociations = yield* deadline.listQueueFleetAssociations.items({ farmId }).pipe(
110 EffectStream.runCollect,
111 Effect.map((chunk) => Array.from(chunk)),
112 Effect.catchTag("ResourceNotFoundException", () =>
113 Effect.succeed([] as deadline.QueueFleetAssociationSummary[]),
114 ),
115 );
116 yield* Effect.forEach(
117 queueFleetAssociations,
118 (assoc) =>
119 Effect.gen(function* () {
120 if (assoc.status !== "STOPPED") {
121 yield* deadline
122 .updateQueueFleetAssociation({
123 farmId,
124 queueId: assoc.queueId,
125 fleetId: assoc.fleetId,
126 status: "STOP_SCHEDULING_AND_CANCEL_TASKS",
127 })
128 .pipe(
129 // Already gone (the association or its farm).
130 Effect.catchTag("ResourceNotFoundException", () => Effect.void),
131 );
132 }
133 yield* retryWhileConflict(
134 deadline.deleteQueueFleetAssociation({
135 farmId,
136 queueId: assoc.queueId,
137 fleetId: assoc.fleetId,
138 }),
139 ).pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void));
140 }),
141 { concurrency: 4, discard: true },
142 );
143
144 // 2. Queue-limit associations — same stop-then-delete dance.
145 const queueLimitAssociations = yield* deadline.listQueueLimitAssociations.items({ farmId }).pipe(
146 EffectStream.runCollect,
147 Effect.map((chunk) => Array.from(chunk)),
148 Effect.catchTag("ResourceNotFoundException", () =>
149 Effect.succeed([] as deadline.QueueLimitAssociationSummary[]),
150 ),
151 );
152 yield* Effect.forEach(
153 queueLimitAssociations,
154 (assoc) =>
155 Effect.gen(function* () {
156 if (assoc.status !== "STOPPED") {
157 yield* deadline
158 .updateQueueLimitAssociation({
159 farmId,
160 queueId: assoc.queueId,
161 limitId: assoc.limitId,
162 status: "STOP_LIMIT_USAGE_AND_CANCEL_TASKS",
163 })
164 .pipe(
165 // Already gone (the association or its farm).
166 Effect.catchTag("ResourceNotFoundException", () => Effect.void),
167 );
168 }
169 yield* retryWhileConflict(
170 deadline.deleteQueueLimitAssociation({
171 farmId,
172 queueId: assoc.queueId,
173 limitId: assoc.limitId,
174 }),
175 ).pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void));
176 }),
177 { concurrency: 4, discard: true },
178 );
179
180 // 3. Budgets.
181 const budgets = yield* deadline.listBudgets.items({ farmId }).pipe(
182 EffectStream.runCollect,
183 Effect.map((chunk) => Array.from(chunk)),
184 Effect.catchTag("ResourceNotFoundException", () =>
185 Effect.succeed([] as deadline.BudgetSummary[]),
186 ),
187 );
188 yield* Effect.forEach(
189 budgets,
190 (budget) =>
191 deadline
192 .deleteBudget({ farmId, budgetId: budget.budgetId })
193 .pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void)),
194 { concurrency: 4, discard: true },
195 );
196
197 // 4. Queues (deletion is asynchronous; the farm delete below retries
198 // through the drain window). A queue's own log group is reaped too.
199 const queues = yield* deadline.listQueues.items({ farmId }).pipe(
200 EffectStream.runCollect,
201 Effect.map((chunk) => Array.from(chunk)),
202 Effect.catchTag("ResourceNotFoundException", () =>
203 Effect.succeed([] as deadline.QueueSummary[]),
204 ),
205 );
206 yield* Effect.forEach(
207 queues,
208 (queue) =>
209 retryWhileConflict(deadline.deleteQueue({ farmId, queueId: queue.queueId })).pipe(
210 Effect.catchTag("ResourceNotFoundException", () => Effect.void),
211 ),
212 { concurrency: 4, discard: true },
213 );
214
215 // 5. Fleets (deletion drains workers asynchronously).
216 const fleets = yield* deadline.listFleets.items({ farmId }).pipe(
217 EffectStream.runCollect,
218 Effect.map((chunk) => Array.from(chunk)),
219 Effect.catchTag("ResourceNotFoundException", () =>
220 Effect.succeed([] as deadline.FleetSummary[]),
221 ),
222 );
223 yield* Effect.forEach(
224 fleets,
225 (fleet) =>
226 retryWhileConflict(deadline.deleteFleet({ farmId, fleetId: fleet.fleetId })).pipe(
227 Effect.catchTag("ResourceNotFoundException", () => Effect.void),
228 ),
229 { concurrency: 4, discard: true },
230 );
231
232 // 6. Limits (their queue-limit associations were deleted above).
233 const limits = yield* deadline.listLimits.items({ farmId }).pipe(
234 EffectStream.runCollect,
235 Effect.map((chunk) => Array.from(chunk)),
236 Effect.catchTag("ResourceNotFoundException", () =>
237 Effect.succeed([] as deadline.LimitSummary[]),
238 ),
239 );
240 yield* Effect.forEach(
241 limits,
242 // deleteLimit is idempotent server-side: neither ResourceNotFound nor
243 // Conflict is in its error union — deleting a missing limit succeeds.
244 (limit) => deadline.deleteLimit({ farmId, limitId: limit.limitId }),
245 { concurrency: 4, discard: true },
246 );
247
248 // 7. Storage profiles (after queues — a queue's allowedStorageProfileIds
249 // reference blocks profile deletion with ConflictException).
250 const storageProfiles = yield* deadline.listStorageProfiles.items({ farmId }).pipe(
251 EffectStream.runCollect,
252 Effect.map((chunk) => Array.from(chunk)),
253 Effect.catchTag("ResourceNotFoundException", () =>
254 Effect.succeed([] as deadline.StorageProfileSummary[]),
255 ),
256 );
257 yield* Effect.forEach(
258 storageProfiles,
259 (profile) =>
260 retryWhileConflict(
261 deadline.deleteStorageProfile({
262 farmId,
263 storageProfileId: profile.storageProfileId,
264 }),
265 ).pipe(Effect.catchTag("ResourceNotFoundException", () => Effect.void)),
266 { concurrency: 4, discard: true },
267 );
268 });
269
270 // Explicitly-typed retry wrappers — an inline `Effect.retry` in provider
271 // lifecycle code leaks `Retry.Return`'s conditional type into declaration
272 // emit and widens the provider layer to `unknown` for every consumer of
273 // `AWS.providers()`.
274
275 /**
276 * Retry through Deadline `ConflictException` (STATUS_CONFLICT while a
277 * sub-resource drains or a concurrent mutation settles). Bounded.
278 */
279 export const retryWhileConflict = <A, E extends { readonly _tag: string }, R>(
280 self: Effect.Effect<A, E, R>,
281 ): Effect.Effect<A, E, R> =>
282 Effect.retry(self, {
283 while: (e) => e._tag === "ConflictException",
284 schedule: Schedule.max([Schedule.spaced("6 seconds"), Schedule.recurs(9)]),
285 });
286
287 /**
288 * A freshly-created farm rejects sub-resource mutations for a short window:
289 * "Farm ... infrastructure setup in progress" surfaces as a 429
290 * `ThrottlingException` or a `ConflictException`, and the farm's internal
291 * components (e.g. its BudgetTracker) surface as `ResourceNotFoundException`
292 * before they are provisioned. Bounded retry through the settling window
293 * (~60s). Genuine throttling is also safely absorbed here.
294 */
295 export const retryWhileFarmSettling = <A, E extends { readonly _tag: string }, R>(
296 self: Effect.Effect<A, E, R>,
297 ): Effect.Effect<A, E, R> =>
298 Effect.retry(self, {
299 while: (e) =>
300 e._tag === "ConflictException" ||
301 e._tag === "InternalServerException" ||
302 e._tag === "ResourceNotFoundException" ||
303 e._tag === "ThrottlingException",
304 schedule: Schedule.max([Schedule.spaced("6 seconds"), Schedule.recurs(9)]),
305 });
306
307 /**
308 * A freshly-created IAM role may not have propagated when Deadline validates
309 * it — createMonitor surfaces this as AccessDeniedException, createFleet /
310 * updateFleet as the typed `DeadlineRoleNotAssumable` ("Could not
311 * sts:AssumeRole ... check its trust policy"). Bounded retry through the
312 * propagation window (~60s).
313 */
314 export const retryThroughIamPropagation = <A, E extends { readonly _tag: string }, R>(
315 self: Effect.Effect<A, E, R>,
316 ): Effect.Effect<A, E, R> =>
317 Effect.retry(self, {
318 while: (e) => e._tag === "AccessDeniedException" || e._tag === "DeadlineRoleNotAssumable",
319 schedule: Schedule.max([Schedule.spaced("6 seconds"), Schedule.recurs(9)]),
320 });