The best way to control Claude Code, Codex and more from any computer or phone

main
ts 545 lines 17.4 KB
Raw
1 import * as dataform from "@distilled.cloud/gcp/dataform_v1";
2 import { createPhysicalName } from "alchemy/PhysicalName";
3 import { tagRecord } from "alchemy/Tags";
4 import * as Data from "effect/Data";
5 import * as Effect from "effect/Effect";
6 import * as Schedule from "effect/Schedule";
7 import * as Stream from "effect/Stream";
8 import { isTransientGcpError } from "../Errors.ts";
9 import {
10 alchemyLabelKeys,
11 createInternalLabels,
12 hasAlchemyLabels,
13 stripInternalLabels,
14 } from "../Labels.ts";
15 import { waitForOperation as waitForGcpOperation } from "../Operation.ts";
16
17 export const MAX_ID_LENGTH = 63;
18 export const MAX_DISPLAY_NAME_LENGTH = 63;
19
20 export class DataformPending extends Data.TaggedError("GCP.Dataform.Pending")<{
21 resourceName: string;
22 }> {}
23
24 export class DataformStillExists extends Data.TaggedError("GCP.Dataform.StillExists")<{
25 resourceName: string;
26 }> {}
27
28 export const lastSegment = (value: string) => {
29 const trimmed = value.replace(/\/+$/, "");
30 const parts = trimmed.split("/");
31 return parts[parts.length - 1] || trimmed;
32 };
33
34 export const locationParent = (project: string, location: string) =>
35 `projects/${project}/locations/${location}`;
36
37 export const normalizeLocation = (location: string | undefined, defaultLocation: string) =>
38 lastSegment(location ?? defaultLocation).toLowerCase();
39
40 export const parseResourceName = (name: string, collection: string, defaultLocation: string) => {
41 const parts = name.split("/").filter((part) => part.length > 0);
42 const collectionAt = parts.lastIndexOf(collection);
43 const locationsAt = parts.lastIndexOf("locations");
44 const projectsAt = parts.lastIndexOf("projects");
45 return {
46 project: projectsAt >= 0 && parts[projectsAt + 1] ? parts[projectsAt + 1]! : "",
47 location:
48 locationsAt >= 0 && parts[locationsAt + 1] ? parts[locationsAt + 1]! : defaultLocation,
49 id: collectionAt >= 0 && parts[collectionAt + 1] ? parts[collectionAt + 1]! : lastSegment(name),
50 parent:
51 collectionAt > 0
52 ? parts.slice(0, collectionAt).join("/")
53 : parts.slice(0, Math.max(0, parts.length - 1)).join("/"),
54 };
55 };
56
57 export const expandParent = (
58 value: string,
59 project: string,
60 location: string,
61 collection: string,
62 ) => {
63 if (value.includes("/")) return value.replace(/\/+$/, "");
64 return `${locationParent(project, location)}/${collection}/${value}`;
65 };
66
67 export const expandRepository = (value: string, project: string, location: string) =>
68 expandParent(value, project, location, "repositories");
69
70 export const rfc1035 = (name: string, maxLength = MAX_ID_LENGTH): string => {
71 let next = name
72 .toLowerCase()
73 .replace(/[^a-z0-9-]/g, "-")
74 .replace(/-+/g, "-")
75 .replace(/^-+|-+$/g, "");
76 if (!/^[a-z]/.test(next)) next = `d${next}`;
77 next = next.slice(0, maxLength).replace(/-+$/g, "");
78 if (next.length === 0) return "dataform";
79 if (!/[a-z0-9]$/.test(next)) next = `${next.slice(0, maxLength - 1)}0`;
80 return next.slice(0, maxLength);
81 };
82
83 export const toPhysicalId = (
84 id: string,
85 explicit: string | undefined,
86 existing: string | undefined,
87 maxLength = MAX_ID_LENGTH,
88 ) =>
89 Effect.gen(function* () {
90 if (explicit !== undefined) return rfc1035(explicit, maxLength);
91 if (existing !== undefined) return existing;
92 return rfc1035(
93 yield* createPhysicalName({
94 id,
95 maxLength,
96 lowercase: true,
97 }),
98 maxLength,
99 );
100 });
101
102 export const userLabels = (
103 labels: Record<string, string | undefined> | null | undefined,
104 ): Record<string, string> => stripInternalLabels(tagRecord(labels));
105
106 export const hasAlchemyLabelMap = (labels: Record<string, string | undefined> | null | undefined) =>
107 Object.keys(labels ?? {}).some((key) => key.startsWith("alchemy-"));
108
109 export const sameText = (left: string | undefined, right: string | undefined) =>
110 (left ?? "") === (right ?? "");
111
112 export const sameJson = (left: unknown, right: unknown) =>
113 JSON.stringify(left ?? null) === JSON.stringify(right ?? null);
114
115 export const updateMaskOf = (...fields: Array<string | undefined>) =>
116 fields.filter((field): field is string => field !== undefined).join(",");
117
118 export const replaceOnIdentity = (input: {
119 previousId?: string;
120 nextId?: string;
121 previousLocation?: string;
122 nextLocation?: string;
123 previousParent?: string;
124 nextParent?: string;
125 extra?: boolean;
126 }) => {
127 const locationChanged =
128 (input.previousLocation ?? "") !== "" &&
129 (input.nextLocation ?? "") !== "" &&
130 lastSegment(input.previousLocation ?? "").toLowerCase() !==
131 lastSegment(input.nextLocation ?? "").toLowerCase();
132 const parentChanged =
133 (input.previousParent ?? "") !== "" &&
134 (input.nextParent ?? "") !== "" &&
135 (input.previousParent ?? "") !== (input.nextParent ?? "");
136 const idChanged =
137 input.previousId !== undefined &&
138 input.nextId !== undefined &&
139 input.previousId !== input.nextId;
140 if (input.extra === true || locationChanged || parentChanged || idChanged) {
141 const samePhysical =
142 !locationChanged &&
143 !parentChanged &&
144 input.previousId !== undefined &&
145 input.nextId === input.previousId;
146 return {
147 action: "replace" as const,
148 deleteFirst: samePhysical,
149 };
150 }
151 return undefined;
152 };
153
154 const markerOf = (stack: string, stage: string, id: string) =>
155 `[alchemy ${alchemyLabelKeys.stack}=${stack} ${alchemyLabelKeys.stage}=${stage} ${alchemyLabelKeys.id}=${id}]`;
156
157 const compactMarkerOf = (stack: string, stage: string, id: string) =>
158 `[alc ${stack} ${stage} ${id}]`;
159
160 const shrinkMarker = (
161 labels: Record<string, string>,
162 maxLength: number,
163 build: (stack: string, stage: string, id: string) => string,
164 ) => {
165 let stack = labels[alchemyLabelKeys.stack] ?? "x";
166 let stage = labels[alchemyLabelKeys.stage] ?? "x";
167 let id = labels[alchemyLabelKeys.id] ?? "x";
168 let marker = build(stack, stage, id);
169 while (marker.length > maxLength && (stack.length > 1 || stage.length > 1 || id.length > 1)) {
170 if (stack.length >= stage.length && stack.length >= id.length) {
171 stack = stack.slice(0, -1);
172 } else if (stage.length >= id.length) {
173 stage = stage.slice(0, -1);
174 } else {
175 id = id.slice(0, -1);
176 }
177 marker = build(stack, stage, id);
178 }
179 return marker.slice(0, maxLength);
180 };
181
182 export const encodeOwnershipLine = (
183 labels: Record<string, string>,
184 text: string | undefined,
185 maxLength = MAX_DISPLAY_NAME_LENGTH,
186 ): string => {
187 const trimmed = text?.replace(/[\r\n]+/g, " ").trim();
188 const reserved =
189 trimmed && trimmed.length > 0 ? Math.min(trimmed.length + 1, Math.max(0, maxLength - 24)) : 0;
190 const budget = Math.max(16, maxLength - reserved);
191 const marker =
192 budget < 54
193 ? shrinkMarker(labels, budget, compactMarkerOf)
194 : shrinkMarker(labels, budget, markerOf);
195 if (!trimmed) return marker.slice(0, maxLength);
196 return `${marker} ${trimmed}`.slice(0, maxLength);
197 };
198
199 export const parseOwnership = (
200 text: string | undefined,
201 ): {
202 labels: Record<string, string>;
203 text: string | undefined;
204 } => {
205 if (text?.startsWith("[alc ")) {
206 const end = text.indexOf("]");
207 if (end < 0) return { labels: {}, text };
208 const parts = text.slice("[alc ".length, end).trim().split(/\s+/);
209 const labels: Record<string, string> = {};
210 if (parts[0]) labels[alchemyLabelKeys.stack] = parts[0]!;
211 if (parts[1]) labels[alchemyLabelKeys.stage] = parts[1]!;
212 if (parts[2]) labels[alchemyLabelKeys.id] = parts[2]!;
213 const rest = text.slice(end + 1).replace(/^[\s\n]+/, "");
214 return { labels, text: rest.length > 0 ? rest : undefined };
215 }
216 if (!text?.startsWith("[alchemy ")) {
217 return { labels: {}, text };
218 }
219 const end = text.indexOf("]");
220 if (end < 0) return { labels: {}, text };
221 const labels: Record<string, string> = {};
222 for (const part of text.slice("[alchemy ".length, end).split(/\s+/)) {
223 const eq = part.indexOf("=");
224 if (eq > 0) {
225 labels[part.slice(0, eq)] = part.slice(eq + 1);
226 }
227 }
228 const rest = text.slice(end + 1).replace(/^[\s\n]+/, "");
229 return { labels, text: rest.length > 0 ? rest : undefined };
230 };
231
232 export const hasOwnershipMarker = (text: string | undefined) =>
233 Object.keys(parseOwnership(text).labels).some((key) => key.startsWith("alchemy-"));
234
235 const prefixMatch = (expected: string, observed: string) =>
236 expected === observed || expected.startsWith(observed) || observed.startsWith(expected);
237
238 export const ownedByAlchemy = (id: string, text: string | undefined) =>
239 Effect.gen(function* () {
240 const expected = yield* createInternalLabels(id);
241 const { labels } = parseOwnership(text);
242 if (!hasOwnershipMarker(text)) return false;
243 const exact = yield* hasAlchemyLabels(id, labels);
244 if (exact) return true;
245 return (
246 prefixMatch(expected[alchemyLabelKeys.stack] ?? "", labels[alchemyLabelKeys.stack] ?? "") &&
247 prefixMatch(expected[alchemyLabelKeys.stage] ?? "", labels[alchemyLabelKeys.stage] ?? "") &&
248 prefixMatch(expected[alchemyLabelKeys.id] ?? "", labels[alchemyLabelKeys.id] ?? "")
249 );
250 });
251
252 export const mergeOwnedVars = (
253 vars: Record<string, string> | undefined,
254 ownership: Record<string, string>,
255 ): Record<string, string> => ({ ...vars, ...ownership });
256
257 export const retryTransient = <A, E extends { readonly _tag: string }, R>(
258 effect: Effect.Effect<A, E, R>,
259 ) =>
260 effect.pipe(
261 Effect.retry({
262 while: isTransientGcpError,
263 times: 8,
264 schedule: Schedule.exponential("250 millis"),
265 }),
266 );
267
268 export class DeleteNotConfirmed extends Data.TaggedError("GCP.Dataform.DeleteNotConfirmed")<{}> {}
269
270 /** Poll until the resource is gone; fails if it is still readable after ~60s. */
271 export const waitUntilGone = <A, E, R>(get: Effect.Effect<A | undefined, E, R>) =>
272 get.pipe(
273 Effect.repeat({
274 schedule: Schedule.spaced("2 seconds"),
275 until: (value) => value === undefined,
276 times: 30,
277 }),
278 Effect.flatMap((value): Effect.Effect<void, DeleteNotConfirmed> =>
279 value === undefined ? Effect.void : Effect.fail(new DeleteNotConfirmed()),
280 ),
281 );
282
283 export const waitUntilExists = <A, E extends { readonly _tag: string }, R>(
284 get: Effect.Effect<A | undefined, E, R>,
285 name: string,
286 ) =>
287 get.pipe(
288 Effect.flatMap((value) =>
289 value !== undefined
290 ? Effect.succeed(value)
291 : Effect.fail(new DataformPending({ resourceName: name })),
292 ),
293 Effect.retry({
294 while: (error) => error._tag === "GCP.Dataform.Pending",
295 times: 8,
296 schedule: Schedule.exponential("250 millis"),
297 }),
298 );
299
300 /**
301 * Wait for a Dataform operation (folder tree deletes, repository moves).
302 * An operation that finished with `ALREADY_EXISTS` (6) is a lost create
303 * race; `notFoundOk` also accepts `NOT_FOUND` (5).
304 */
305 export const waitForOperation = (
306 operation: dataform.Operation,
307 options?: { notFoundOk?: boolean },
308 ) =>
309 waitForGcpOperation(
310 operation,
311 (name) =>
312 dataform
313 .getProjectsLocationsOperations({ name })
314 .pipe(
315 Effect.catchTag("NotFound", (error) =>
316 options?.notFoundOk === true
317 ? Effect.succeed<dataform.Operation>({ name, done: true })
318 : Effect.fail(error),
319 ),
320 ),
321 { budget: "10 minutes" },
322 ).pipe(
323 Effect.catchIf(
324 (error) =>
325 error._tag === "GCP.OperationFailed" &&
326 (error.code === 6 || (options?.notFoundOk === true && error.code === 5)),
327 () => Effect.void,
328 ),
329 Effect.asVoid,
330 );
331
332 export const collectPages = <Page, Item, E extends { readonly _tag: string }, R>(
333 pages: Stream.Stream<Page, E, R>,
334 items: (page: Page) => readonly Item[] | null | undefined,
335 ) =>
336 pages.pipe(
337 Stream.flatMap((page) => Stream.fromIterable(items(page) ?? [])),
338 Stream.runCollect,
339 Effect.map((chunk) => Array.from(chunk)),
340 );
341
342 const emptyList = <A>() => Effect.succeed<A[]>([]);
343
344 const emptyOnMissing = <A, E extends { readonly _tag: string }, R>(
345 effect: Effect.Effect<A[], E, R>,
346 ) =>
347 effect.pipe(
348 Effect.catchIf(
349 (error) => error._tag === "NotFound",
350 () => emptyList<A>(),
351 ),
352 );
353
354 export const listRepositories = (project: string, location: string) =>
355 emptyOnMissing(
356 collectPages(
357 dataform.listProjectsLocationsRepositories.pages({
358 parent: locationParent(project, location),
359 pageSize: 1000,
360 }),
361 (page) => page.repositories ?? [],
362 ).pipe(
363 Effect.catchIf(
364 () => true,
365 () =>
366 collectPages(
367 dataform.listProjectsLocationsRepositories.pages({
368 parent: `projects/${project}/locations/-`,
369 pageSize: 1000,
370 }),
371 (page) => page.repositories ?? [],
372 ),
373 ),
374 ),
375 );
376
377 export const listOwnedRepositories = (project: string, location: string) =>
378 listRepositories(project, location).pipe(
379 Effect.map((repos) => repos.filter((repo) => hasAlchemyLabelMap(repo.labels))),
380 );
381
382 export const listTeamFolders = (project: string, location: string) =>
383 emptyOnMissing(
384 collectPages(
385 dataform.searchProjectsLocationsTeamFolders.pages({
386 location: locationParent(project, location),
387 pageSize: 1000,
388 }),
389 (page) =>
390 (page.results ?? []).flatMap((result) => (result.teamFolder ? [result.teamFolder] : [])),
391 ),
392 );
393
394 export const listFolders = (project: string, location: string) =>
395 Effect.gen(function* () {
396 const parent = locationParent(project, location);
397 const seen = new Set<string>();
398 const folders: dataform.Folder[] = [];
399 const queue: string[] = [];
400
401 const ingest = (folder: dataform.Folder | undefined) => {
402 const name = folder?.name;
403 if (folder === undefined || name === undefined || seen.has(name)) {
404 return;
405 }
406 seen.add(name);
407 folders.push(folder);
408 queue.push(name);
409 };
410
411 const root = yield* emptyOnMissing(
412 collectPages(
413 dataform.queryUserRootContentsProjectsLocations.pages({
414 location: parent,
415 pageSize: 1000,
416 }),
417 (page) => page.entries ?? [],
418 ),
419 );
420 for (const entry of root) ingest(entry.folder);
421
422 const teams = yield* listTeamFolders(project, location);
423 for (const team of teams) {
424 if (team.name) queue.push(team.name);
425 }
426
427 let steps = 0;
428 while (queue.length > 0 && steps < 50) {
429 steps += 1;
430 const name = queue.shift()!;
431 const entries = name.includes("/teamFolders/")
432 ? yield* emptyOnMissing(
433 collectPages(
434 dataform.queryContentsProjectsLocationsTeamFolders.pages({
435 teamFolder: name,
436 pageSize: 1000,
437 }),
438 (page) => page.entries ?? [],
439 ),
440 )
441 : yield* emptyOnMissing(
442 collectPages(
443 dataform.queryFolderContentsProjectsLocationsFolders.pages({
444 folder: name,
445 pageSize: 1000,
446 }),
447 (page) => page.entries ?? [],
448 ),
449 );
450 for (const entry of entries) ingest(entry.folder);
451 }
452
453 return folders;
454 });
455
456 export const listReleaseConfigs = (parent: string) =>
457 parent.length === 0
458 ? emptyList<dataform.ReleaseConfig>()
459 : emptyOnMissing(
460 collectPages(
461 dataform.listProjectsLocationsRepositoriesReleaseConfigs.pages({
462 parent,
463 pageSize: 1000,
464 }),
465 (page) => page.releaseConfigs ?? [],
466 ),
467 );
468
469 export const listWorkflowConfigs = (parent: string) =>
470 parent.length === 0
471 ? emptyList<dataform.WorkflowConfig>()
472 : emptyOnMissing(
473 collectPages(
474 dataform.listProjectsLocationsRepositoriesWorkflowConfigs.pages({
475 parent,
476 pageSize: 1000,
477 }),
478 (page) => page.workflowConfigs ?? [],
479 ),
480 );
481
482 export const listWorkspaces = (parent: string) =>
483 parent.length === 0
484 ? emptyList<dataform.Workspace>()
485 : emptyOnMissing(
486 collectPages(
487 dataform.listProjectsLocationsRepositoriesWorkspaces.pages({
488 parent,
489 pageSize: 1000,
490 }),
491 (page) => page.workspaces ?? [],
492 ),
493 );
494
495 export const listWorkflowInvocations = (parent: string) =>
496 parent.length === 0
497 ? emptyList<dataform.WorkflowInvocation>()
498 : emptyOnMissing(
499 collectPages(
500 dataform.listProjectsLocationsRepositoriesWorkflowInvocations.pages({
501 parent,
502 pageSize: 1000,
503 }),
504 (page) => page.workflowInvocations ?? [],
505 ),
506 );
507
508 export const forEachOwnedRepository = <A, E, R>(
509 project: string,
510 location: string,
511 fn: (repo: dataform.Repository) => Effect.Effect<readonly A[], E, R>,
512 ) =>
513 listOwnedRepositories(project, location).pipe(
514 Effect.flatMap((repos) =>
515 Effect.forEach(repos, fn, { concurrency: 4 }).pipe(Effect.map((chunks) => chunks.flat())),
516 ),
517 );
518
519 export const deleteFolderTree = (name: string) =>
520 dataform.deleteProjectsLocationsFolders({ name }).pipe(
521 Effect.catchTag("Conflict", () =>
522 dataform
523 .deleteTreeProjectsLocationsFolders({
524 name,
525 body: { force: true },
526 })
527 .pipe(Effect.flatMap((operation) => waitForOperation(operation, { notFoundOk: true }))),
528 ),
529 Effect.catchTag("NotFound", () => Effect.void),
530 Effect.asVoid,
531 );
532
533 export const deleteTeamFolderTree = (name: string) =>
534 dataform.deleteProjectsLocationsTeamFolders({ name }).pipe(
535 Effect.catchTag("Conflict", () =>
536 dataform
537 .deleteTreeProjectsLocationsTeamFolders({
538 name,
539 body: { force: true },
540 })
541 .pipe(Effect.flatMap((operation) => waitForOperation(operation, { notFoundOk: true }))),
542 ),
543 Effect.catchTag("NotFound", () => Effect.void),
544 Effect.asVoid,
545 );