The best way to control Claude Code, Codex and more from any computer or phone

main
ts 399 lines 14.3 KB
Raw
1 import * as dataplex from "@distilled.cloud/gcp/dataplex_v1";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import { tagRecord } from "alchemy/Tags";
7 import * as Effect from "effect/Effect";
8 import * as Schedule from "effect/Schedule";
9 import { GcpEnvironment } from "../Environment.ts";
10 import { createInternalLabels, diffLabels, hasAlchemyLabels, toLabels } from "../Labels.ts";
11 import type { Providers } from "../Providers.ts";
12 import {
13 DataplexNotResolved,
14 collectPages,
15 fingerprint,
16 hasAlchemyLabelMap,
17 listAtLocation,
18 normalizeLocation,
19 parentOf,
20 parseName,
21 replaceOnIdentity,
22 retryQuota,
23 toPhysicalId,
24 userLabels,
25 waitForOperation,
26 waitUntilExists,
27 waitUntilGone,
28 } from "./internal.ts";
29
30 export type DataScanDataSource = dataplex.GoogleCloudDataplexV1DataSource;
31 export type DataScanExecutionSpec = dataplex.GoogleCloudDataplexV1DataScanExecutionSpec;
32 export type DataScanExecutionIdentity = dataplex.GoogleCloudDataplexV1ExecutionIdentity;
33 export type DataQualitySpec = dataplex.GoogleCloudDataplexV1DataQualitySpec;
34 export type DataProfileSpec = dataplex.GoogleCloudDataplexV1DataProfileSpec;
35 export type DataDiscoverySpec = dataplex.GoogleCloudDataplexV1DataDiscoverySpec;
36 export type DataDocumentationSpec = dataplex.GoogleCloudDataplexV1DataDocumentationSpec;
37 export type UnstructuredDataProfileSpec = dataplex.GoogleCloudDataplexV1UnstructuredDataProfileSpec;
38
39 export type DataScanProps = {
40 /**
41 * DataScan id. If omitted, a unique name is generated. Must contain
42 * only lowercase letters, numbers and hyphens, start with a letter,
43 * end with a letter or number, and be 1-63 characters. Immutable —
44 * changing it replaces the scan.
45 */
46 dataScanId?: string;
47 /**
48 * Region (`us-central1`, …). Immutable — changing it replaces the
49 * scan.
50 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
51 */
52 location?: string;
53 /**
54 * Description (1-1024 characters).
55 */
56 description?: string;
57 /**
58 * User-friendly display name (1-256 characters).
59 */
60 displayName?: string;
61 /**
62 * User labels. Alchemy ownership labels are merged in automatically.
63 */
64 labels?: Record<string, string>;
65 /**
66 * Immutable data source (Dataplex entity or BigQuery / GCS resource).
67 * Changing it replaces the scan.
68 */
69 data: DataScanDataSource;
70 /**
71 * Execution settings. Defaults to on-demand.
72 */
73 executionSpec?: DataScanExecutionSpec;
74 /**
75 * Immutable identity used to run the scan. Changing it replaces the
76 * scan.
77 */
78 executionIdentity?: DataScanExecutionIdentity;
79 /**
80 * Data quality scan settings. Mutually exclusive with the other spec
81 * fields.
82 */
83 dataQualitySpec?: DataQualitySpec;
84 /**
85 * Data profile scan settings.
86 */
87 dataProfileSpec?: DataProfileSpec;
88 /**
89 * Data discovery scan settings.
90 */
91 dataDiscoverySpec?: DataDiscoverySpec;
92 /**
93 * Data documentation scan settings.
94 */
95 dataDocumentationSpec?: DataDocumentationSpec;
96 /**
97 * Unstructured data profile scan settings.
98 */
99 unstructuredDataProfileSpec?: UnstructuredDataProfileSpec;
100 };
101
102 export type DataScan = Resource<
103 "GCP.Dataplex.DataScan",
104 DataScanProps,
105 {
106 /** Full resource name. */
107 name: string;
108 /** DataScan id (last path segment). */
109 dataScanId: string;
110 /** Project id. */
111 project: string;
112 /** Location id. */
113 location: string;
114 /** Description. */
115 description: string | undefined;
116 /** Display name. */
117 displayName: string | undefined;
118 /** User labels (Alchemy ownership labels stripped). */
119 labels: Record<string, string>;
120 /** Current state. */
121 state: string | undefined;
122 /** Data source. */
123 data: DataScanDataSource | undefined;
124 /** Execution spec. */
125 executionSpec: DataScanExecutionSpec | undefined;
126 /** Scan type. */
127 type: string | undefined;
128 /** Data quality spec, if any. */
129 dataQualitySpec: DataQualitySpec | undefined;
130 /** Data profile spec, if any. */
131 dataProfileSpec: DataProfileSpec | undefined;
132 /** Data discovery spec, if any. */
133 dataDiscoverySpec: DataDiscoverySpec | undefined;
134 /** Data documentation spec, if any. */
135 dataDocumentationSpec: DataDocumentationSpec | undefined;
136 /** Unstructured data profile spec, if any. */
137 unstructuredDataProfileSpec: UnstructuredDataProfileSpec | undefined;
138 /** System uid. */
139 uid: string | undefined;
140 /** RFC3339 creation timestamp. */
141 createTime: string | undefined;
142 /** RFC3339 last-update timestamp. */
143 updateTime: string | undefined;
144 },
145 never,
146 Providers
147 >;
148
149 /**
150 * A Dataplex DataScan that profiles, quality-checks, discovers, or
151 * documents a BigQuery table, dataset, or Cloud Storage bucket.
152 *
153 * Location, scan id, data source, and execution identity are immutable.
154 * Description, display name, labels, execution spec, and scan specs
155 * update in place.
156 *
157 * ### Creating a DataScan
158 * **Example:** On-demand profile scan of a BigQuery table
159 * ```typescript
160 * const scan = yield* GCP.Dataplex.DataScan("OrdersProfile", {
161 * displayName: "orders profile",
162 * labels: { env: "test" },
163 * data: {
164 * resource:
165 * "//bigquery.googleapis.com/projects/my-project/datasets/sales/tables/orders",
166 * },
167 * dataProfileSpec: {},
168 * });
169 * ```
170 *
171 * @resource
172 * @category Dataplex
173 */
174 export const DataScan = Resource<DataScan>("GCP.Dataplex.DataScan");
175
176 const resourceName = (project: string, location: string, dataScanId: string) =>
177 `projects/${project}/locations/${location}/dataScans/${dataScanId}`;
178
179 const toAttrs = (scan: dataplex.GoogleCloudDataplexV1DataScan, project: string) => {
180 const name = scan.name ?? "";
181 const parsed = parseName(name, "dataScans");
182 return {
183 name,
184 dataScanId: parsed.id,
185 project: parsed.project || project,
186 location: parsed.location,
187 description: scan.description,
188 displayName: scan.displayName,
189 labels: userLabels(scan.labels),
190 state: scan.state,
191 data: scan.data,
192 executionSpec: scan.executionSpec,
193 type: scan.type,
194 dataQualitySpec: scan.dataQualitySpec,
195 dataProfileSpec: scan.dataProfileSpec,
196 dataDiscoverySpec: scan.dataDiscoverySpec,
197 dataDocumentationSpec: scan.dataDocumentationSpec,
198 unstructuredDataProfileSpec: scan.unstructuredDataProfileSpec,
199 uid: scan.uid,
200 createTime: scan.createTime,
201 updateTime: scan.updateTime,
202 };
203 };
204
205 const getByName = (name: string) =>
206 retryQuota(dataplex.getProjectsLocationsDataScans({ name, view: "FULL" })).pipe(
207 Effect.catchTag("NotFound", () => Effect.undefined),
208 );
209
210 const listScans = (project: string, region: string) => {
211 const collect = (parent: string) =>
212 collectPages(
213 dataplex.listProjectsLocationsDataScans.pages({
214 parent,
215 pageSize: 1000,
216 }),
217 (page) => page.dataScans,
218 ).pipe(
219 Effect.map((items) => items.filter((item) => hasAlchemyLabelMap(item.labels))),
220 Effect.catchTag("NotFound", () => Effect.succeed([])),
221 );
222 return listAtLocation(project, region, collect);
223 };
224
225 const dataSourceOf = (data: DataScanDataSource | undefined) => ({
226 entity: data?.entity ?? "",
227 resource: data?.resource ?? "",
228 });
229
230 export const DataScanProvider = () =>
231 Provider.succeed(DataScan, {
232 stables: ["name", "dataScanId", "project", "location", "uid", "createTime"],
233
234 diff: Effect.fn(function* ({ news, olds, output }) {
235 if (!isResolved(news)) return undefined;
236 const env = yield* GcpEnvironment.current;
237 return replaceOnIdentity({
238 previousId: olds?.dataScanId ?? output?.dataScanId,
239 nextId: news.dataScanId ?? olds?.dataScanId ?? output?.dataScanId,
240 previousLocation: normalizeLocation(olds?.location ?? output?.location, env.region),
241 nextLocation: normalizeLocation(
242 news.location ?? olds?.location ?? output?.location,
243 env.region,
244 ),
245 extra:
246 fingerprint(dataSourceOf(news.data)) !==
247 fingerprint(dataSourceOf(olds?.data ?? output?.data)) ||
248 fingerprint(news.executionIdentity ?? olds?.executionIdentity) !==
249 fingerprint(olds?.executionIdentity),
250 });
251 }),
252
253 read: Effect.fn(function* ({ id, olds, output }) {
254 const env = yield* GcpEnvironment.current;
255 const dataScanId = yield* toPhysicalId(id, olds?.dataScanId, output?.dataScanId, "datascan");
256 const location = normalizeLocation(olds?.location ?? output?.location, env.region);
257 const name = output?.name ?? resourceName(env.project, location, dataScanId);
258 const existing = yield* getByName(name);
259 if (existing === undefined) return undefined;
260 const attrs = toAttrs(existing, env.project);
261 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
262 }),
263
264 list: () =>
265 Effect.gen(function* () {
266 const env = yield* GcpEnvironment.current;
267 const items = yield* listScans(env.project, env.region);
268 return items.map((item) => toAttrs(item, env.project));
269 }),
270
271 reconcile: Effect.fn(function* ({ id, news, output }) {
272 const env = yield* GcpEnvironment.current;
273 const dataScanId = yield* toPhysicalId(id, news.dataScanId, output?.dataScanId, "datascan");
274 const location = normalizeLocation(news.location ?? output?.location, env.region);
275 const name = resourceName(env.project, location, dataScanId);
276 const desiredLabels = {
277 ...toLabels(news.labels),
278 ...(yield* createInternalLabels(id)),
279 };
280
281 let current = yield* getByName(output?.name ?? name);
282
283 if (current === undefined) {
284 const created = yield* retryQuota(
285 dataplex.createProjectsLocationsDataScans({
286 parent: parentOf(env.project, location),
287 dataScanId,
288 body: {
289 description: news.description,
290 displayName: news.displayName,
291 labels: desiredLabels,
292 data: news.data,
293 executionSpec: news.executionSpec,
294 executionIdentity: news.executionIdentity,
295 dataQualitySpec: news.dataQualitySpec,
296 dataProfileSpec: news.dataProfileSpec,
297 dataDiscoverySpec: news.dataDiscoverySpec,
298 dataDocumentationSpec: news.dataDocumentationSpec,
299 unstructuredDataProfileSpec: news.unstructuredDataProfileSpec,
300 },
301 }),
302 ).pipe(Effect.catchTag("Conflict", () => Effect.undefined));
303 if (created !== undefined) {
304 yield* waitForOperation(created);
305 }
306 current = yield* waitUntilExists(getByName(name), name);
307 }
308
309 if (current === undefined) {
310 return yield* new DataplexNotResolved({ resourceName: name });
311 }
312
313 const observedLabels = tagRecord(current.labels);
314 const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
315 const labelsChanged = upsert.length > 0 || removed.length > 0;
316 const descriptionChanged = (current.description ?? "") !== (news.description ?? "");
317 const displayNameChanged = (current.displayName ?? "") !== (news.displayName ?? "");
318 const executionChanged =
319 fingerprint(current.executionSpec) !== fingerprint(news.executionSpec);
320 const qualityChanged =
321 fingerprint(current.dataQualitySpec) !== fingerprint(news.dataQualitySpec);
322 const profileChanged =
323 fingerprint(current.dataProfileSpec) !== fingerprint(news.dataProfileSpec);
324 const discoveryChanged =
325 fingerprint(current.dataDiscoverySpec) !== fingerprint(news.dataDiscoverySpec);
326 const documentationChanged =
327 fingerprint(current.dataDocumentationSpec) !== fingerprint(news.dataDocumentationSpec);
328 const unstructuredChanged =
329 fingerprint(current.unstructuredDataProfileSpec) !==
330 fingerprint(news.unstructuredDataProfileSpec);
331
332 if (
333 labelsChanged ||
334 descriptionChanged ||
335 displayNameChanged ||
336 executionChanged ||
337 qualityChanged ||
338 profileChanged ||
339 discoveryChanged ||
340 documentationChanged ||
341 unstructuredChanged
342 ) {
343 const operation = yield* retryQuota(
344 dataplex.patchProjectsLocationsDataScans({
345 name: current.name ?? name,
346 updateMask: [
347 labelsChanged ? "labels" : undefined,
348 descriptionChanged ? "description" : undefined,
349 displayNameChanged ? "displayName" : undefined,
350 executionChanged ? "executionSpec" : undefined,
351 qualityChanged ? "dataQualitySpec" : undefined,
352 profileChanged ? "dataProfileSpec" : undefined,
353 discoveryChanged ? "dataDiscoverySpec" : undefined,
354 documentationChanged ? "dataDocumentationSpec" : undefined,
355 unstructuredChanged ? "unstructuredDataProfileSpec" : undefined,
356 ]
357 .filter((field): field is string => field !== undefined)
358 .join(","),
359 body: {
360 name: current.name ?? name,
361 description: news.description,
362 displayName: news.displayName,
363 labels: desiredLabels,
364 executionSpec: news.executionSpec,
365 dataQualitySpec: news.dataQualitySpec,
366 dataProfileSpec: news.dataProfileSpec,
367 dataDiscoverySpec: news.dataDiscoverySpec,
368 dataDocumentationSpec: news.dataDocumentationSpec,
369 unstructuredDataProfileSpec: news.unstructuredDataProfileSpec,
370 },
371 }),
372 );
373 yield* waitForOperation(operation);
374 current = yield* waitUntilExists(getByName(current.name ?? name), current.name ?? name);
375 }
376
377 return toAttrs(current, env.project);
378 }),
379
380 delete: Effect.fn(function* ({ output }) {
381 const operation = yield* retryQuota(
382 dataplex.deleteProjectsLocationsDataScans({
383 name: output.name,
384 force: true,
385 }),
386 ).pipe(
387 Effect.retry({
388 while: (error) => error._tag === "Conflict",
389 times: 8,
390 schedule: Schedule.spaced("2 seconds"),
391 }),
392 Effect.catchTag("NotFound", () => Effect.undefined),
393 );
394 if (operation !== undefined) {
395 yield* waitForOperation(operation, { notFoundOk: true });
396 }
397 yield* waitUntilGone(getByName(output.name), output.name);
398 }),
399 });