The best way to control Claude Code, Codex and more from any computer or phone

main
ts 286 lines 9.47 KB
Raw
1 import * as datalineage from "@distilled.cloud/gcp/datalineage_v1";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import * as Data from "effect/Data";
7 import * as Effect from "effect/Effect";
8 import { GcpEnvironment } from "../Environment.ts";
9 import type { Providers } from "../Providers.ts";
10 import {
11 createOwnership,
12 desiredAttributes,
13 findOwnedProcess,
14 listOwnedProcesses,
15 locationParent,
16 normalizeLocation,
17 ownedByAlchemy,
18 parseName,
19 replaceOnIdentity,
20 sameJson,
21 sameText,
22 toPhysicalId,
23 userAttributes,
24 waitForOperation,
25 waitUntilGone,
26 } from "./internal.ts";
27
28 export type ProcessOrigin = {
29 /**
30 * Origin system. Use `CUSTOM` unless reporting lineage on behalf of a
31 * Google system. Non-`CUSTOM` sources may be restricted and billed.
32 * @default "CUSTOM"
33 */
34 sourceType?: datalineage.GoogleCloudDatacatalogLineageV1OriginSourceTypeEnum | (string & {});
35 /**
36 * Origin name. For `CUSTOM`, any identifier. For Google systems, a
37 * resource name in the same project and location.
38 */
39 name?: string;
40 };
41
42 export type ProcessProps = {
43 /**
44 * Process id (the `{process}` segment of
45 * `projects/{project}/locations/{location}/processes/{process}`). If
46 * omitted, a unique name is generated from the stack, stage, and logical
47 * id. Max 200 characters: letters, digits, underscores, hyphens, colons,
48 * periods. Immutable — changing it replaces the process.
49 */
50 processId?: string;
51 /**
52 * Region (`us-central1`, …). Immutable — changing it replaces the
53 * process. `US-CENTRAL1` is accepted and normalized to `us-central1`.
54 * @default the stack's GCP region (`GCP.Region`, profile region, `us-central1`)
55 */
56 location?: string;
57 /**
58 * Human-readable name shown in UIs. Max 200 characters.
59 */
60 displayName?: string;
61 /**
62 * Origin of this process and its runs. Defaults to `{ sourceType:
63 * "CUSTOM", name: "alchemy" }`.
64 */
65 origin?: ProcessOrigin;
66 /**
67 * Non-semantic attributes (classifying or labeling the process). Max
68 * 100 entries. Processes have no labels field, so Alchemy stamps
69 * `alchemy-stack` / `alchemy-stage` / `alchemy-id` here for `list` /
70 * nuke and strips them from attributes.
71 */
72 attributes?: Record<string, unknown>;
73 };
74
75 export type Process = Resource<
76 "GCP.DataLineage.Process",
77 ProcessProps,
78 {
79 /** Full resource name `projects/{project}/locations/{location}/processes/{process}`. */
80 name: string;
81 /** Process id (last path segment). */
82 processId: string;
83 /** Project id. */
84 project: string;
85 /** Location id. */
86 location: string;
87 /** Human-readable display name. */
88 displayName: string | undefined;
89 /** Origin of the process. */
90 origin: ProcessOrigin | undefined;
91 /** User attributes (Alchemy ownership keys stripped). */
92 attributes: Record<string, unknown>;
93 },
94 never,
95 Providers
96 >;
97
98 /**
99 * A Data Lineage process — the definition of a data transformation.
100 *
101 * Processes have no labels field, so Alchemy stamps ownership into
102 * `attributes` for `list` / nuke. `processId` and `location` are identity
103 * — changing either replaces the process. Display name, origin, and
104 * attributes update in place.
105 *
106 * ### Creating a Process
107 * **Example:** Generated name
108 * ```typescript
109 * const process = yield* GCP.DataLineage.Process("Etl", {
110 * displayName: "nightly etl",
111 * attributes: { env: "dev" },
112 * });
113 * ```
114 *
115 * **Example:** Named process with a custom origin
116 * ```typescript
117 * const process = yield* GCP.DataLineage.Process("Etl", {
118 * processId: "orders-etl",
119 * location: "us-central1",
120 * displayName: "orders etl",
121 * origin: { sourceType: "CUSTOM", name: "dbt" },
122 * attributes: { env: "prod" },
123 * });
124 * ```
125 *
126 * @resource
127 * @category DataLineage
128 */
129 export const Process = Resource<Process>("GCP.DataLineage.Process");
130
131 export class ProcessNotResolved extends Data.TaggedError("GCP.DataLineage.ProcessNotResolved")<{
132 resourceName: string;
133 }> {}
134
135 const resourceName = (project: string, location: string, processId: string) =>
136 `${locationParent(project, location)}/processes/${processId}`;
137
138 const defaultOrigin = (
139 origin: ProcessOrigin | undefined,
140 ): datalineage.GoogleCloudDatacatalogLineageV1Origin => ({
141 sourceType: origin?.sourceType ?? "CUSTOM",
142 name: origin?.name ?? "alchemy",
143 });
144
145 const toAttrs = (
146 process: datalineage.GoogleCloudDatacatalogLineageV1Process,
147 project: string,
148 region: string,
149 ) => {
150 const name = process.name ?? "";
151 const parsed = parseName(name, "processes", region);
152 return {
153 name,
154 processId: parsed.id,
155 project: parsed.project || project,
156 location: parsed.location,
157 displayName: process.displayName,
158 origin: process.origin
159 ? {
160 sourceType: process.origin.sourceType,
161 name: process.origin.name,
162 }
163 : undefined,
164 attributes: userAttributes(process.attributes),
165 };
166 };
167
168 const getByName = (name: string) =>
169 name.length === 0
170 ? Effect.undefined
171 : datalineage
172 .getProjectsLocationsProcesses({ name })
173 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
174
175 export const ProcessProvider = () =>
176 Provider.succeed(Process, {
177 stables: ["name", "processId", "project", "location"],
178
179 diff: Effect.fn(function* ({ news, olds, output }) {
180 if (!isResolved(news)) return undefined;
181 const env = yield* GcpEnvironment.current;
182 const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region);
183 const nextLocation = normalizeLocation(
184 news.location ?? olds?.location ?? output?.location,
185 env.region,
186 );
187 return replaceOnIdentity({
188 previousId: olds?.processId ?? output?.processId,
189 nextId: news.processId,
190 extra: output !== undefined && previousLocation !== nextLocation,
191 });
192 }),
193
194 read: Effect.fn(function* ({ id, olds, output }) {
195 const env = yield* GcpEnvironment.current;
196 const location = normalizeLocation(olds?.location ?? output?.location, env.region);
197 const processId = yield* toPhysicalId(id, olds?.processId, output?.processId);
198 const name = output?.name ?? resourceName(env.project, location, processId);
199 const existing =
200 (yield* getByName(name)) ?? (yield* findOwnedProcess(id, env.project, location));
201 if (existing === undefined) return undefined;
202 const attrs = toAttrs(existing, env.project, env.region);
203 return (yield* ownedByAlchemy(id, existing.attributes)) ? attrs : Unowned(attrs);
204 }),
205
206 list: () =>
207 Effect.gen(function* () {
208 const env = yield* GcpEnvironment.current;
209 const rows = yield* listOwnedProcesses(env.project, env.region);
210 return rows.map((row) => toAttrs(row, env.project, env.region));
211 }),
212
213 reconcile: Effect.fn(function* ({ id, news, output }) {
214 const env = yield* GcpEnvironment.current;
215 const location = normalizeLocation(news.location ?? output?.location, env.region);
216 const processId = yield* toPhysicalId(id, news.processId, output?.processId);
217 const name = output?.name ?? resourceName(env.project, location, processId);
218 const ownership = yield* createOwnership(id);
219 const attributes = desiredAttributes(news.attributes, ownership);
220
221 let current =
222 (yield* getByName(name)) ?? (yield* findOwnedProcess(id, env.project, location));
223
224 if (current === undefined) {
225 const created = yield* datalineage
226 .createProjectsLocationsProcesses({
227 parent: locationParent(env.project, location),
228 body: {
229 name,
230 displayName: news.displayName,
231 origin: defaultOrigin(news.origin),
232 attributes,
233 },
234 })
235 .pipe(
236 Effect.catchTag("Conflict", () =>
237 getByName(name).pipe(
238 Effect.flatMap((row) =>
239 row ? Effect.succeed(row) : findOwnedProcess(id, env.project, location),
240 ),
241 ),
242 ),
243 );
244 current = created ?? undefined;
245 }
246
247 if (current === undefined) {
248 return yield* new ProcessNotResolved({ resourceName: name });
249 }
250
251 const resolvedName = current.name ?? name;
252 const desiredOrigin = defaultOrigin(news.origin ?? current.origin);
253 const displayName = news.displayName ?? current.displayName;
254 const originChanged = !sameJson(current.origin, desiredOrigin);
255 const displayNameChanged = !sameText(current.displayName, displayName);
256 const attributesChanged = !sameJson(current.attributes, attributes);
257
258 if (originChanged || displayNameChanged || attributesChanged) {
259 current = yield* datalineage.patchProjectsLocationsProcesses({
260 name: resolvedName,
261 body: {
262 name: resolvedName,
263 displayName,
264 origin: desiredOrigin,
265 attributes,
266 },
267 });
268 }
269
270 return toAttrs(current, env.project, env.region);
271 }),
272
273 delete: Effect.fn(function* ({ output }) {
274 if (!output.name) return;
275 const operation = yield* datalineage
276 .deleteProjectsLocationsProcesses({
277 name: output.name,
278 allowMissing: true,
279 })
280 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
281 if (operation !== undefined) {
282 yield* waitForOperation(operation, { notFoundOk: true });
283 }
284 yield* waitUntilGone(getByName(output.name), output.name);
285 }),
286 });