@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 336 lines 10.8 KB
Raw
1 import * as datalineage from "@distilled.cloud/gcp/datalineage_v1";
2 import { createPhysicalName } from "alchemy/PhysicalName";
3 import * as Data from "effect/Data";
4 import * as Effect from "effect/Effect";
5 import * as Schedule from "effect/Schedule";
6 import * as Stream from "effect/Stream";
7 import { createInternalLabels, hasAlchemyLabels, stripInternalLabels } from "../Labels.ts";
8 import { waitForOperation as waitForGcpOperation } from "../Operation.ts";
9
10 export const MAX_ID_LENGTH = 200;
11 export const MAX_DISPLAY_NAME_LENGTH = 200;
12
13 export class DatalineageNotResolved extends Data.TaggedError(
14 "GCP.DataLineage.ResourceNotResolved",
15 )<{
16 resourceName: string;
17 }> {}
18
19 export class DatalineageStillExists extends Data.TaggedError(
20 "GCP.DataLineage.ResourceStillExists",
21 )<{
22 resourceName: string;
23 }> {}
24
25 export const lastSegment = (value: string) => {
26 const trimmed = value.replace(/\/+$/, "");
27 const parts = trimmed.split("/");
28 return parts[parts.length - 1] || trimmed;
29 };
30
31 export const normalizeLocation = (location: string | undefined, fallback: string) =>
32 lastSegment(location ?? fallback).toLowerCase();
33
34 export const locationParent = (project: string, location: string) =>
35 `projects/${project}/locations/${lastSegment(location).toLowerCase()}`;
36
37 export const parentOf = (name: string) => {
38 const parts = name.split("/").filter((part) => part.length > 0);
39 return parts.slice(0, Math.max(0, parts.length - 2)).join("/");
40 };
41
42 export const parseName = (name: string, collection: string, fallbackLocation: string) => {
43 const parts = name.split("/").filter((part) => part.length > 0);
44 const collectionAt = parts.lastIndexOf(collection);
45 const locationsAt = parts.lastIndexOf("locations");
46 const projectsAt = parts.lastIndexOf("projects");
47 return {
48 project: projectsAt >= 0 && parts[projectsAt + 1] ? parts[projectsAt + 1]! : "",
49 location:
50 locationsAt >= 0 && parts[locationsAt + 1] ? parts[locationsAt + 1]! : fallbackLocation,
51 id: collectionAt >= 0 && parts[collectionAt + 1] ? parts[collectionAt + 1]! : lastSegment(name),
52 parent: collectionAt > 0 ? parts.slice(0, collectionAt).join("/") : parentOf(name),
53 };
54 };
55
56 export const expandParent = (
57 value: string,
58 project: string,
59 location: string,
60 collection: string,
61 ) => {
62 if (value.includes("/")) return value.replace(/\/+$/, "");
63 return `projects/${project}/locations/${location}/${collection}/${value}`;
64 };
65
66 export const processOf = (value: string, project: string, location: string) =>
67 expandParent(value, project, location, "processes");
68
69 export const runOf = (value: string, project: string, location: string) =>
70 expandParent(value, project, location, "runs");
71
72 export const sanitizeId = (name: string, fallback = "lineage"): string => {
73 let next = name
74 .toLowerCase()
75 .replace(/[^a-z0-9_\-.]/g, "-")
76 .replace(/-+/g, "-")
77 .replace(/^-+|-+$/g, "");
78 if (next.length === 0) return fallback;
79 if (!/^[a-z0-9]/.test(next)) next = `p${next}`;
80 next = next.slice(0, MAX_ID_LENGTH).replace(/[-.]+$/g, "");
81 return next.length > 0 ? next : fallback;
82 };
83
84 export const toPhysicalId = (
85 id: string,
86 explicit: string | undefined,
87 existing: string | undefined,
88 fallback = "lineage",
89 ) =>
90 Effect.gen(function* () {
91 if (explicit !== undefined) return sanitizeId(explicit, fallback);
92 if (existing !== undefined) return existing;
93 return sanitizeId(
94 yield* createPhysicalName({
95 id,
96 maxLength: 63,
97 lowercase: true,
98 }),
99 fallback,
100 );
101 });
102
103 export const sameText = (left: string | undefined, right: string | undefined) =>
104 (left ?? "") === (right ?? "");
105
106 export const canonical = (value: unknown): unknown => {
107 if (value === undefined || value === null) return undefined;
108 if (typeof value === "boolean" || typeof value === "number") return value;
109 if (typeof value === "string") {
110 return value.length === 0 ? undefined : value;
111 }
112 if (Array.isArray(value)) {
113 const items = value.map(canonical).filter((item) => item !== undefined);
114 return items.length === 0 ? undefined : items;
115 }
116 if (typeof value === "object") {
117 const entries = Object.entries(value as Record<string, unknown>)
118 .map(([key, item]) => [key, canonical(item)] as const)
119 .filter(([, item]) => item !== undefined)
120 .sort(([left], [right]) => left.localeCompare(right));
121 if (entries.length === 0) return undefined;
122 return Object.fromEntries(entries);
123 }
124 return undefined;
125 };
126
127 export const fingerprint = (value: unknown): string => JSON.stringify(canonical(value) ?? null);
128
129 export const sameJson = (left: unknown, right: unknown) => fingerprint(left) === fingerprint(right);
130
131 export const replaceOnIdentity = (input: {
132 previousId?: string;
133 nextId?: string;
134 previousParent?: string;
135 nextParent?: string;
136 extra?: boolean;
137 }) => {
138 if (input.extra === true) {
139 return { action: "replace" as const, deleteFirst: false };
140 }
141 if (
142 input.previousId !== undefined &&
143 input.nextId !== undefined &&
144 input.previousId !== input.nextId
145 ) {
146 return { action: "replace" as const, deleteFirst: true };
147 }
148 if (
149 input.previousParent !== undefined &&
150 input.nextParent !== undefined &&
151 input.previousParent !== input.nextParent
152 ) {
153 return { action: "replace" as const, deleteFirst: false };
154 }
155 return undefined;
156 };
157
158 export const attributeTags = (
159 attributes: datalineage.DocumentMap | null | undefined,
160 ): Record<string, string> => {
161 const tags: Record<string, string> = {};
162 for (const [key, value] of Object.entries(attributes ?? {})) {
163 if (value === undefined) continue;
164 tags[key] = typeof value === "string" ? value : JSON.stringify(value);
165 }
166 return tags;
167 };
168
169 export const userAttributes = (
170 attributes: datalineage.DocumentMap | null | undefined,
171 ): Record<string, unknown> => {
172 const next: Record<string, unknown> = {};
173 for (const [key, value] of Object.entries(stripInternalLabels(attributeTags(attributes)))) {
174 const raw = attributes?.[key];
175 next[key] = raw !== undefined ? raw : value;
176 }
177 return next;
178 };
179
180 export const desiredAttributes = (
181 attributes: Record<string, unknown> | undefined,
182 ownership: Record<string, string>,
183 ): datalineage.DocumentMap => ({
184 ...attributes,
185 ...ownership,
186 });
187
188 export const hasAlchemyAttributeMap = (attributes: datalineage.DocumentMap | null | undefined) =>
189 Object.keys(attributes ?? {}).some((key) => key.startsWith("alchemy-"));
190
191 export const ownedByAlchemy = (
192 id: string,
193 attributes: datalineage.DocumentMap | null | undefined,
194 ) => hasAlchemyLabels(id, attributeTags(attributes));
195
196 export const collectPages = <Page, Item, E, R>(
197 pages: Stream.Stream<Page, E, R>,
198 items: (page: Page) => readonly Item[] | null | undefined,
199 ) =>
200 pages.pipe(
201 Stream.flatMap((page) => Stream.fromIterable(items(page) ?? [])),
202 Stream.runCollect,
203 Effect.map((chunk) => Array.from(chunk) as Item[]),
204 );
205
206 const emptyList = <A>() => Effect.succeed<A[]>([]);
207
208 const emptyOnMissing = <A, E extends { readonly _tag: string }, R>(
209 effect: Effect.Effect<A[], E, R>,
210 ) =>
211 effect.pipe(
212 Effect.catchIf(
213 (error) => error._tag === "NotFound",
214 () => emptyList<A>(),
215 ),
216 );
217
218 export const listProcesses = (parent: string) =>
219 parent.length === 0
220 ? emptyList<datalineage.GoogleCloudDatacatalogLineageV1Process>()
221 : emptyOnMissing(
222 collectPages(
223 datalineage.listProjectsLocationsProcesses.pages({
224 parent,
225 pageSize: 100,
226 }),
227 (page) => page.processes,
228 ),
229 );
230
231 export const listRuns = (parent: string) =>
232 parent.length === 0
233 ? emptyList<datalineage.GoogleCloudDatacatalogLineageV1Run>()
234 : emptyOnMissing(
235 collectPages(
236 datalineage.listProjectsLocationsProcessesRuns.pages({
237 parent,
238 pageSize: 100,
239 }),
240 (page) => page.runs,
241 ),
242 );
243
244 export const listLineageEvents = (parent: string) =>
245 parent.length === 0
246 ? emptyList<datalineage.GoogleCloudDatacatalogLineageV1LineageEvent>()
247 : emptyOnMissing(
248 collectPages(
249 datalineage.listProjectsLocationsProcessesRunsLineageEvents.pages({
250 parent,
251 pageSize: 100,
252 }),
253 (page) => page.lineageEvents,
254 ),
255 );
256
257 export const listOwnedProcesses = (project: string, location: string) =>
258 listProcesses(locationParent(project, location)).pipe(
259 Effect.map((rows) => rows.filter((row) => hasAlchemyAttributeMap(row.attributes))),
260 );
261
262 export const findOwnedProcess = (id: string, project: string, location: string) =>
263 Effect.gen(function* () {
264 const rows = yield* listProcesses(locationParent(project, location));
265 for (const row of rows) {
266 if (yield* ownedByAlchemy(id, row.attributes)) return row;
267 }
268 return undefined;
269 });
270
271 export const findOwnedRun = (id: string, process: string) =>
272 Effect.gen(function* () {
273 const rows = yield* listRuns(process);
274 for (const row of rows) {
275 if (yield* ownedByAlchemy(id, row.attributes)) return row;
276 }
277 return undefined;
278 });
279
280 /** Wait for an operation through the shared GCP waiter. */
281 export const waitForOperation = (
282 operation: datalineage.GoogleLongrunningOperation,
283 options?: { notFoundOk?: boolean; alreadyExistsOk?: boolean },
284 ) =>
285 waitForGcpOperation(
286 operation,
287 (name) =>
288 datalineage.getProjectsLocationsOperations({ name }).pipe(
289 Effect.catchTag("NotFound", (error) =>
290 options?.notFoundOk === true
291 ? Effect.succeed<datalineage.GoogleLongrunningOperation>({
292 name,
293 done: true,
294 })
295 : Effect.fail(error),
296 ),
297 ),
298 { budget: "10 minutes" },
299 ).pipe(
300 // ALREADY_EXISTS (6): a concurrent create won the race. NOT_FOUND (5)
301 // is success for a delete.
302 Effect.catchIf(
303 (error) =>
304 error._tag === "GCP.OperationFailed" &&
305 ((options?.alreadyExistsOk !== false && error.code === 6) ||
306 (options?.notFoundOk === true && error.code === 5)),
307 () => Effect.void,
308 ),
309 // Re-read the finished operation for its typed response and metadata.
310 Effect.flatMap(() =>
311 operation.name === undefined || operation.name.length === 0
312 ? Effect.succeed(operation)
313 : datalineage
314 .getProjectsLocationsOperations({ name: operation.name })
315 .pipe(Effect.catchTag("NotFound", () => Effect.succeed(operation))),
316 ),
317 );
318
319 export const waitUntilGone = <A, E, R>(
320 get: Effect.Effect<A | undefined, E, R>,
321 name: string,
322 ): Effect.Effect<void, E | DatalineageStillExists, R> =>
323 get.pipe(
324 Effect.filterOrFail(
325 (value) => value === undefined,
326 () => new DatalineageStillExists({ resourceName: name }),
327 ),
328 Effect.retry({
329 while: (error) => error instanceof DatalineageStillExists,
330 times: 10,
331 schedule: Schedule.spaced("1 second"),
332 }),
333 Effect.asVoid,
334 );
335
336 export const createOwnership = (id: string) => createInternalLabels(id);