The best way to control Claude Code, Codex and more from any computer or phone

main
ts 369 lines 12.1 KB
Raw
1 import * as datalineage from "@distilled.cloud/gcp/datalineage_v1";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import * as Data from "effect/Data";
7 import * as Effect from "effect/Effect";
8 import { GcpEnvironment } from "../Environment.ts";
9 import type { Providers } from "../Providers.ts";
10 import {
11 fingerprint,
12 hasAlchemyAttributeMap,
13 listLineageEvents,
14 listOwnedProcesses,
15 listRuns,
16 normalizeLocation,
17 parseName,
18 processOf,
19 replaceOnIdentity,
20 toPhysicalId,
21 waitUntilGone,
22 } from "./internal.ts";
23
24 export type LineageEntityReference = {
25 /**
26 * Fully qualified name of the entity, for example
27 * `bigquery:project.dataset.table` or `custom:project.system.table`.
28 */
29 fullyQualifiedName: string;
30 /**
31 * Optional nested field path within the entity. Empty for asset-level
32 * lineage.
33 */
34 field?: string[];
35 };
36
37 export type LineageEventLink = {
38 /** Source entity. */
39 source: LineageEntityReference;
40 /** Target entity. */
41 target: LineageEntityReference;
42 /**
43 * How the target depends on the source (`EXACT_COPY` or `OTHER`).
44 */
45 dependencyType?:
46 | datalineage.GoogleCloudDatacatalogLineageV1DependencyInfoDependencyTypeEnum
47 | (string & {});
48 };
49
50 export type ProcessesRunsLineageEventProps = {
51 /**
52 * Parent run. Full name
53 * `projects/{project}/locations/{location}/processes/{process}/runs/{run}`
54 * or a run id combined with `process`. Immutable — changing it replaces
55 * the event.
56 */
57 run: string;
58 /**
59 * Parent process used when `run` is a bare id. Full process name or
60 * process id combined with `location`.
61 */
62 process?: string;
63 /**
64 * Event id (the `{lineage_event}` segment). If omitted, a unique name
65 * is generated. Max 200 characters: letters, digits, underscores,
66 * hyphens, colons, periods. Immutable — changing it replaces the event.
67 */
68 lineageEventId?: string;
69 /**
70 * Region used when `run` / `process` is a bare id.
71 * @default the stack's GCP region (`GCP.Region`, profile region, `us-central1`)
72 */
73 location?: string;
74 /**
75 * RFC3339 start of the transformation. Required by the API; Alchemy
76 * reuses the observed value on subsequent reconciles when omitted.
77 */
78 startTime?: string;
79 /**
80 * RFC3339 end of the transformation.
81 */
82 endTime?: string;
83 /**
84 * Source-target links (max 100). Lineage events have no labels or
85 * attributes field, so Alchemy lists events under alchemy-owned
86 * parent processes for nuke.
87 */
88 links?: LineageEventLink[];
89 };
90
91 export type ProcessesRunsLineageEvent = Resource<
92 "GCP.DataLineage.ProcessesRunsLineageEvent",
93 ProcessesRunsLineageEventProps,
94 {
95 /** Full resource name `.../runs/{run}/lineageEvents/{lineage_event}`. */
96 name: string;
97 /** Lineage event id (last path segment). */
98 lineageEventId: string;
99 /** Parent run resource name. */
100 run: string;
101 /** Parent process resource name. */
102 process: string;
103 /** Project id. */
104 project: string;
105 /** Location id. */
106 location: string;
107 /** RFC3339 start time. */
108 startTime: string | undefined;
109 /** RFC3339 end time. */
110 endTime: string | undefined;
111 /** Source-target links. */
112 links: LineageEventLink[];
113 },
114 never,
115 Providers
116 >;
117
118 /**
119 * A Data Lineage event — one operation that moved data from source
120 * assets to target assets.
121 *
122 * Events are immutable after create (no patch API). Identity is the
123 * parent run plus `lineageEventId`. Changing links or timestamps
124 * replaces the event. Events have no labels, so `list` / nuke walks
125 * alchemy-owned parent processes.
126 *
127 * ### Creating a Lineage Event
128 * **Example:** One source-to-target link
129 * ```typescript
130 * const event = yield* GCP.DataLineage.ProcessesRunsLineageEvent("Load", {
131 * run: run.name,
132 * startTime: "2024-01-01T00:00:00Z",
133 * links: [
134 * {
135 * source: { fullyQualifiedName: "custom:proj.raw.orders" },
136 * target: { fullyQualifiedName: "custom:proj.dw.orders" },
137 * },
138 * ],
139 * });
140 * ```
141 *
142 * @resource
143 * @category DataLineage
144 */
145 export const ProcessesRunsLineageEvent = Resource<ProcessesRunsLineageEvent>(
146 "GCP.DataLineage.ProcessesRunsLineageEvent",
147 );
148
149 /** Alias matching the Data Lineage resource name. */
150 export const LineageEvent = ProcessesRunsLineageEvent;
151 export type LineageEvent = ProcessesRunsLineageEvent;
152
153 export class ProcessesRunsLineageEventNotResolved extends Data.TaggedError(
154 "GCP.DataLineage.ProcessesRunsLineageEventNotResolved",
155 )<{
156 resourceName: string;
157 }> {}
158
159 const resourceName = (run: string, lineageEventId: string) =>
160 `${run}/lineageEvents/${lineageEventId}`;
161
162 const nowIso = () => Effect.sync(() => new Date().toISOString());
163
164 const runOf = (run: string, process: string | undefined, project: string, location: string) => {
165 if (run.includes("/")) return run.replace(/\/+$/, "");
166 const parent = processOf(process ?? "", project, location);
167 return `${parent}/runs/${run}`;
168 };
169
170 const linksOf = (
171 links: readonly LineageEventLink[] | undefined,
172 ): datalineage.GoogleCloudDatacatalogLineageV1EventLink[] =>
173 (links ?? []).map((link) => ({
174 source: {
175 fullyQualifiedName: link.source.fullyQualifiedName,
176 field: link.source.field,
177 },
178 target: {
179 fullyQualifiedName: link.target.fullyQualifiedName,
180 field: link.target.field,
181 },
182 dependencyInfo:
183 link.dependencyType !== undefined ? { dependencyType: link.dependencyType } : undefined,
184 }));
185
186 const linksFrom = (
187 links: readonly datalineage.GoogleCloudDatacatalogLineageV1EventLink[] | undefined,
188 ): LineageEventLink[] =>
189 (links ?? [])
190 .filter(
191 (link) =>
192 (link.source?.fullyQualifiedName ?? "").length > 0 &&
193 (link.target?.fullyQualifiedName ?? "").length > 0,
194 )
195 .map((link) => ({
196 source: {
197 fullyQualifiedName: link.source?.fullyQualifiedName ?? "",
198 field: link.source?.field,
199 },
200 target: {
201 fullyQualifiedName: link.target?.fullyQualifiedName ?? "",
202 field: link.target?.field,
203 },
204 dependencyType: link.dependencyInfo?.dependencyType,
205 }));
206
207 const toAttrs = (
208 event: datalineage.GoogleCloudDatacatalogLineageV1LineageEvent,
209 project: string,
210 region: string,
211 ) => {
212 const name = event.name ?? "";
213 const parsed = parseName(name, "lineageEvents", region);
214 const process = parseName(parsed.parent, "runs", region).parent;
215 return {
216 name,
217 lineageEventId: parsed.id,
218 run: parsed.parent,
219 process,
220 project: parsed.project || project,
221 location: parsed.location,
222 startTime: event.startTime,
223 endTime: event.endTime,
224 links: linksFrom(event.links),
225 };
226 };
227
228 const getByName = (name: string) =>
229 name.length === 0
230 ? Effect.undefined
231 : datalineage
232 .getProjectsLocationsProcessesRunsLineageEvents({ name })
233 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
234
235 const getProcess = (name: string) =>
236 name.length === 0
237 ? Effect.undefined
238 : datalineage
239 .getProjectsLocationsProcesses({ name })
240 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
241
242 const identityFingerprint = (news: {
243 startTime?: string;
244 endTime?: string;
245 links?: LineageEventLink[];
246 }) =>
247 fingerprint({
248 startTime: news.startTime,
249 endTime: news.endTime,
250 links: linksOf(news.links),
251 });
252
253 export const ProcessesRunsLineageEventProvider = () =>
254 Provider.succeed(ProcessesRunsLineageEvent, {
255 stables: ["name", "lineageEventId", "run", "process", "project", "location"],
256
257 diff: Effect.fn(function* ({ news, olds, output }) {
258 if (!isResolved(news)) return undefined;
259 const env = yield* GcpEnvironment.current;
260 const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region);
261 const nextLocation = normalizeLocation(
262 news.location ?? olds?.location ?? output?.location,
263 env.region,
264 );
265 const previous = identityFingerprint({
266 startTime: olds?.startTime ?? output?.startTime,
267 endTime: olds?.endTime ?? output?.endTime,
268 links: olds?.links ?? output?.links,
269 });
270 const next = identityFingerprint({
271 startTime: news.startTime ?? olds?.startTime ?? output?.startTime,
272 endTime: news.endTime ?? olds?.endTime ?? output?.endTime,
273 links: news.links ?? olds?.links ?? output?.links,
274 });
275 return replaceOnIdentity({
276 previousId: olds?.lineageEventId ?? output?.lineageEventId,
277 nextId: news.lineageEventId,
278 previousParent: olds?.run ?? output?.run,
279 nextParent: news.run,
280 extra:
281 (output !== undefined && previousLocation !== nextLocation) ||
282 (output !== undefined && previous !== next),
283 });
284 }),
285
286 read: Effect.fn(function* ({ id, olds, output }) {
287 const env = yield* GcpEnvironment.current;
288 const parentRef = olds?.run ?? output?.run;
289 // A recovery read of an interrupted create gets persisted props whose
290 // unresolved parent reference was stripped; without the parent there
291 // is no name to look up, so report nothing found rather than calling
292 // the API with an empty path segment.
293 if (output?.name === undefined && (typeof parentRef !== "string" || parentRef.length === 0)) {
294 return undefined;
295 }
296 const location = normalizeLocation(olds?.location ?? output?.location, env.region);
297 const run = runOf(
298 olds?.run ?? output?.run ?? "",
299 olds?.process ?? output?.process,
300 env.project,
301 location,
302 );
303 const lineageEventId = yield* toPhysicalId(id, olds?.lineageEventId, output?.lineageEventId);
304 const name = output?.name ?? resourceName(run, lineageEventId);
305 const existing = yield* getByName(name);
306 if (existing === undefined) return undefined;
307 const attrs = toAttrs(existing, env.project, env.region);
308 const parent = yield* getProcess(attrs.process);
309 return parent === undefined || hasAlchemyAttributeMap(parent.attributes)
310 ? attrs
311 : Unowned(attrs);
312 }),
313
314 list: () =>
315 Effect.gen(function* () {
316 const env = yield* GcpEnvironment.current;
317 const processes = yield* listOwnedProcesses(env.project, env.region);
318 const runs = (yield* Effect.forEach(processes, (process) => listRuns(process.name ?? ""), {
319 concurrency: 4,
320 })).flat();
321 const events = (yield* Effect.forEach(runs, (run) => listLineageEvents(run.name ?? ""), {
322 concurrency: 4,
323 })).flat();
324 return events.map((event) => toAttrs(event, env.project, env.region));
325 }),
326
327 reconcile: Effect.fn(function* ({ id, news, output }) {
328 const env = yield* GcpEnvironment.current;
329 const location = normalizeLocation(news.location ?? output?.location, env.region);
330 const run = runOf(news.run, news.process, env.project, location);
331 const lineageEventId = yield* toPhysicalId(id, news.lineageEventId, output?.lineageEventId);
332 const name = output?.name ?? resourceName(run, lineageEventId);
333
334 let current = yield* getByName(name);
335
336 if (current === undefined) {
337 const startTime = news.startTime ?? (yield* nowIso());
338 const created = yield* datalineage
339 .createProjectsLocationsProcessesRunsLineageEvents({
340 parent: run,
341 body: {
342 name,
343 startTime,
344 endTime: news.endTime,
345 links: linksOf(news.links),
346 },
347 })
348 .pipe(Effect.catchTag("Conflict", () => getByName(name)));
349 current = created ?? undefined;
350 }
351
352 if (current === undefined) {
353 return yield* new ProcessesRunsLineageEventNotResolved({ resourceName: name });
354 }
355
356 return toAttrs(current, env.project, env.region);
357 }),
358
359 delete: Effect.fn(function* ({ output }) {
360 if (!output.name) return;
361 yield* datalineage
362 .deleteProjectsLocationsProcessesRunsLineageEvents({
363 name: output.name,
364 allowMissing: true,
365 })
366 .pipe(Effect.catchTag("NotFound", () => Effect.void));
367 yield* waitUntilGone(getByName(output.name), output.name);
368 }),
369 });