The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 04:45 UTC
| 1 | import * as dataplex from "@distilled.cloud/gcp/dataplex_v1"; |
| 2 | import { Unowned } from "alchemy/AdoptPolicy"; |
| 3 | import { isResolved } from "alchemy/Diff"; |
| 4 | import * as Provider from "alchemy/Provider"; |
| 5 | import { Resource } from "alchemy/Resource"; |
| 6 | import { tagRecord } from "alchemy/Tags"; |
| 7 | import * as Effect from "effect/Effect"; |
| 8 | import * as Schedule from "effect/Schedule"; |
| 9 | import { GcpEnvironment } from "../Environment.ts"; |
| 10 | import { createInternalLabels, diffLabels, hasAlchemyLabels, toLabels } from "../Labels.ts"; |
| 11 | import type { Providers } from "../Providers.ts"; |
| 12 | import { |
| 13 | DataplexNotResolved, |
| 14 | collectPages, |
| 15 | fingerprint, |
| 16 | hasAlchemyLabelMap, |
| 17 | listAtLocation, |
| 18 | normalizeLocation, |
| 19 | parentOf, |
| 20 | parseName, |
| 21 | replaceOnIdentity, |
| 22 | retryQuota, |
| 23 | toPhysicalId, |
| 24 | userLabels, |
| 25 | waitForOperation, |
| 26 | waitUntilExists, |
| 27 | waitUntilGone, |
| 28 | } from "./internal.ts"; |
| 29 | |
| 30 | export type DataScanDataSource = dataplex.GoogleCloudDataplexV1DataSource; |
| 31 | export type DataScanExecutionSpec = dataplex.GoogleCloudDataplexV1DataScanExecutionSpec; |
| 32 | export type DataScanExecutionIdentity = dataplex.GoogleCloudDataplexV1ExecutionIdentity; |
| 33 | export type DataQualitySpec = dataplex.GoogleCloudDataplexV1DataQualitySpec; |
| 34 | export type DataProfileSpec = dataplex.GoogleCloudDataplexV1DataProfileSpec; |
| 35 | export type DataDiscoverySpec = dataplex.GoogleCloudDataplexV1DataDiscoverySpec; |
| 36 | export type DataDocumentationSpec = dataplex.GoogleCloudDataplexV1DataDocumentationSpec; |
| 37 | export type UnstructuredDataProfileSpec = dataplex.GoogleCloudDataplexV1UnstructuredDataProfileSpec; |
| 38 | |
| 39 | export type DataScanProps = { |
| 40 | /** |
| 41 | * DataScan id. If omitted, a unique name is generated. Must contain |
| 42 | * only lowercase letters, numbers and hyphens, start with a letter, |
| 43 | * end with a letter or number, and be 1-63 characters. Immutable — |
| 44 | * changing it replaces the scan. |
| 45 | */ |
| 46 | dataScanId?: string; |
| 47 | /** |
| 48 | * Region (`us-central1`, …). Immutable — changing it replaces the |
| 49 | * scan. |
| 50 | * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`) |
| 51 | */ |
| 52 | location?: string; |
| 53 | /** |
| 54 | * Description (1-1024 characters). |
| 55 | */ |
| 56 | description?: string; |
| 57 | /** |
| 58 | * User-friendly display name (1-256 characters). |
| 59 | */ |
| 60 | displayName?: string; |
| 61 | /** |
| 62 | * User labels. Alchemy ownership labels are merged in automatically. |
| 63 | */ |
| 64 | labels?: Record<string, string>; |
| 65 | /** |
| 66 | * Immutable data source (Dataplex entity or BigQuery / GCS resource). |
| 67 | * Changing it replaces the scan. |
| 68 | */ |
| 69 | data: DataScanDataSource; |
| 70 | /** |
| 71 | * Execution settings. Defaults to on-demand. |
| 72 | */ |
| 73 | executionSpec?: DataScanExecutionSpec; |
| 74 | /** |
| 75 | * Immutable identity used to run the scan. Changing it replaces the |
| 76 | * scan. |
| 77 | */ |
| 78 | executionIdentity?: DataScanExecutionIdentity; |
| 79 | /** |
| 80 | * Data quality scan settings. Mutually exclusive with the other spec |
| 81 | * fields. |
| 82 | */ |
| 83 | dataQualitySpec?: DataQualitySpec; |
| 84 | /** |
| 85 | * Data profile scan settings. |
| 86 | */ |
| 87 | dataProfileSpec?: DataProfileSpec; |
| 88 | /** |
| 89 | * Data discovery scan settings. |
| 90 | */ |
| 91 | dataDiscoverySpec?: DataDiscoverySpec; |
| 92 | /** |
| 93 | * Data documentation scan settings. |
| 94 | */ |
| 95 | dataDocumentationSpec?: DataDocumentationSpec; |
| 96 | /** |
| 97 | * Unstructured data profile scan settings. |
| 98 | */ |
| 99 | unstructuredDataProfileSpec?: UnstructuredDataProfileSpec; |
| 100 | }; |
| 101 | |
| 102 | export type DataScan = Resource< |
| 103 | "GCP.Dataplex.DataScan", |
| 104 | DataScanProps, |
| 105 | { |
| 106 | /** Full resource name. */ |
| 107 | name: string; |
| 108 | /** DataScan id (last path segment). */ |
| 109 | dataScanId: string; |
| 110 | /** Project id. */ |
| 111 | project: string; |
| 112 | /** Location id. */ |
| 113 | location: string; |
| 114 | /** Description. */ |
| 115 | description: string | undefined; |
| 116 | /** Display name. */ |
| 117 | displayName: string | undefined; |
| 118 | /** User labels (Alchemy ownership labels stripped). */ |
| 119 | labels: Record<string, string>; |
| 120 | /** Current state. */ |
| 121 | state: string | undefined; |
| 122 | /** Data source. */ |
| 123 | data: DataScanDataSource | undefined; |
| 124 | /** Execution spec. */ |
| 125 | executionSpec: DataScanExecutionSpec | undefined; |
| 126 | /** Scan type. */ |
| 127 | type: string | undefined; |
| 128 | /** Data quality spec, if any. */ |
| 129 | dataQualitySpec: DataQualitySpec | undefined; |
| 130 | /** Data profile spec, if any. */ |
| 131 | dataProfileSpec: DataProfileSpec | undefined; |
| 132 | /** Data discovery spec, if any. */ |
| 133 | dataDiscoverySpec: DataDiscoverySpec | undefined; |
| 134 | /** Data documentation spec, if any. */ |
| 135 | dataDocumentationSpec: DataDocumentationSpec | undefined; |
| 136 | /** Unstructured data profile spec, if any. */ |
| 137 | unstructuredDataProfileSpec: UnstructuredDataProfileSpec | undefined; |
| 138 | /** System uid. */ |
| 139 | uid: string | undefined; |
| 140 | /** RFC3339 creation timestamp. */ |
| 141 | createTime: string | undefined; |
| 142 | /** RFC3339 last-update timestamp. */ |
| 143 | updateTime: string | undefined; |
| 144 | }, |
| 145 | never, |
| 146 | Providers |
| 147 | >; |
| 148 | |
| 149 | /** |
| 150 | * A Dataplex DataScan that profiles, quality-checks, discovers, or |
| 151 | * documents a BigQuery table, dataset, or Cloud Storage bucket. |
| 152 | * |
| 153 | * Location, scan id, data source, and execution identity are immutable. |
| 154 | * Description, display name, labels, execution spec, and scan specs |
| 155 | * update in place. |
| 156 | * |
| 157 | * ### Creating a DataScan |
| 158 | * **Example:** On-demand profile scan of a BigQuery table |
| 159 | * ```typescript |
| 160 | * const scan = yield* GCP.Dataplex.DataScan("OrdersProfile", { |
| 161 | * displayName: "orders profile", |
| 162 | * labels: { env: "test" }, |
| 163 | * data: { |
| 164 | * resource: |
| 165 | * "//bigquery.googleapis.com/projects/my-project/datasets/sales/tables/orders", |
| 166 | * }, |
| 167 | * dataProfileSpec: {}, |
| 168 | * }); |
| 169 | * ``` |
| 170 | * |
| 171 | * @resource |
| 172 | * @category Dataplex |
| 173 | */ |
| 174 | export const DataScan = Resource<DataScan>("GCP.Dataplex.DataScan"); |
| 175 | |
| 176 | const resourceName = (project: string, location: string, dataScanId: string) => |
| 177 | `projects/${project}/locations/${location}/dataScans/${dataScanId}`; |
| 178 | |
| 179 | const toAttrs = (scan: dataplex.GoogleCloudDataplexV1DataScan, project: string) => { |
| 180 | const name = scan.name ?? ""; |
| 181 | const parsed = parseName(name, "dataScans"); |
| 182 | return { |
| 183 | name, |
| 184 | dataScanId: parsed.id, |
| 185 | project: parsed.project || project, |
| 186 | location: parsed.location, |
| 187 | description: scan.description, |
| 188 | displayName: scan.displayName, |
| 189 | labels: userLabels(scan.labels), |
| 190 | state: scan.state, |
| 191 | data: scan.data, |
| 192 | executionSpec: scan.executionSpec, |
| 193 | type: scan.type, |
| 194 | dataQualitySpec: scan.dataQualitySpec, |
| 195 | dataProfileSpec: scan.dataProfileSpec, |
| 196 | dataDiscoverySpec: scan.dataDiscoverySpec, |
| 197 | dataDocumentationSpec: scan.dataDocumentationSpec, |
| 198 | unstructuredDataProfileSpec: scan.unstructuredDataProfileSpec, |
| 199 | uid: scan.uid, |
| 200 | createTime: scan.createTime, |
| 201 | updateTime: scan.updateTime, |
| 202 | }; |
| 203 | }; |
| 204 | |
| 205 | const getByName = (name: string) => |
| 206 | retryQuota(dataplex.getProjectsLocationsDataScans({ name, view: "FULL" })).pipe( |
| 207 | Effect.catchTag("NotFound", () => Effect.undefined), |
| 208 | ); |
| 209 | |
| 210 | const listScans = (project: string, region: string) => { |
| 211 | const collect = (parent: string) => |
| 212 | collectPages( |
| 213 | dataplex.listProjectsLocationsDataScans.pages({ |
| 214 | parent, |
| 215 | pageSize: 1000, |
| 216 | }), |
| 217 | (page) => page.dataScans, |
| 218 | ).pipe( |
| 219 | Effect.map((items) => items.filter((item) => hasAlchemyLabelMap(item.labels))), |
| 220 | Effect.catchTag("NotFound", () => Effect.succeed([])), |
| 221 | ); |
| 222 | return listAtLocation(project, region, collect); |
| 223 | }; |
| 224 | |
| 225 | const dataSourceOf = (data: DataScanDataSource | undefined) => ({ |
| 226 | entity: data?.entity ?? "", |
| 227 | resource: data?.resource ?? "", |
| 228 | }); |
| 229 | |
| 230 | export const DataScanProvider = () => |
| 231 | Provider.succeed(DataScan, { |
| 232 | stables: ["name", "dataScanId", "project", "location", "uid", "createTime"], |
| 233 | |
| 234 | diff: Effect.fn(function* ({ news, olds, output }) { |
| 235 | if (!isResolved(news)) return undefined; |
| 236 | const env = yield* GcpEnvironment.current; |
| 237 | return replaceOnIdentity({ |
| 238 | previousId: olds?.dataScanId ?? output?.dataScanId, |
| 239 | nextId: news.dataScanId ?? olds?.dataScanId ?? output?.dataScanId, |
| 240 | previousLocation: normalizeLocation(olds?.location ?? output?.location, env.region), |
| 241 | nextLocation: normalizeLocation( |
| 242 | news.location ?? olds?.location ?? output?.location, |
| 243 | env.region, |
| 244 | ), |
| 245 | extra: |
| 246 | fingerprint(dataSourceOf(news.data)) !== |
| 247 | fingerprint(dataSourceOf(olds?.data ?? output?.data)) || |
| 248 | fingerprint(news.executionIdentity ?? olds?.executionIdentity) !== |
| 249 | fingerprint(olds?.executionIdentity), |
| 250 | }); |
| 251 | }), |
| 252 | |
| 253 | read: Effect.fn(function* ({ id, olds, output }) { |
| 254 | const env = yield* GcpEnvironment.current; |
| 255 | const dataScanId = yield* toPhysicalId(id, olds?.dataScanId, output?.dataScanId, "datascan"); |
| 256 | const location = normalizeLocation(olds?.location ?? output?.location, env.region); |
| 257 | const name = output?.name ?? resourceName(env.project, location, dataScanId); |
| 258 | const existing = yield* getByName(name); |
| 259 | if (existing === undefined) return undefined; |
| 260 | const attrs = toAttrs(existing, env.project); |
| 261 | return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs); |
| 262 | }), |
| 263 | |
| 264 | list: () => |
| 265 | Effect.gen(function* () { |
| 266 | const env = yield* GcpEnvironment.current; |
| 267 | const items = yield* listScans(env.project, env.region); |
| 268 | return items.map((item) => toAttrs(item, env.project)); |
| 269 | }), |
| 270 | |
| 271 | reconcile: Effect.fn(function* ({ id, news, output }) { |
| 272 | const env = yield* GcpEnvironment.current; |
| 273 | const dataScanId = yield* toPhysicalId(id, news.dataScanId, output?.dataScanId, "datascan"); |
| 274 | const location = normalizeLocation(news.location ?? output?.location, env.region); |
| 275 | const name = resourceName(env.project, location, dataScanId); |
| 276 | const desiredLabels = { |
| 277 | ...toLabels(news.labels), |
| 278 | ...(yield* createInternalLabels(id)), |
| 279 | }; |
| 280 | |
| 281 | let current = yield* getByName(output?.name ?? name); |
| 282 | |
| 283 | if (current === undefined) { |
| 284 | const created = yield* retryQuota( |
| 285 | dataplex.createProjectsLocationsDataScans({ |
| 286 | parent: parentOf(env.project, location), |
| 287 | dataScanId, |
| 288 | body: { |
| 289 | description: news.description, |
| 290 | displayName: news.displayName, |
| 291 | labels: desiredLabels, |
| 292 | data: news.data, |
| 293 | executionSpec: news.executionSpec, |
| 294 | executionIdentity: news.executionIdentity, |
| 295 | dataQualitySpec: news.dataQualitySpec, |
| 296 | dataProfileSpec: news.dataProfileSpec, |
| 297 | dataDiscoverySpec: news.dataDiscoverySpec, |
| 298 | dataDocumentationSpec: news.dataDocumentationSpec, |
| 299 | unstructuredDataProfileSpec: news.unstructuredDataProfileSpec, |
| 300 | }, |
| 301 | }), |
| 302 | ).pipe(Effect.catchTag("Conflict", () => Effect.undefined)); |
| 303 | if (created !== undefined) { |
| 304 | yield* waitForOperation(created); |
| 305 | } |
| 306 | current = yield* waitUntilExists(getByName(name), name); |
| 307 | } |
| 308 | |
| 309 | if (current === undefined) { |
| 310 | return yield* new DataplexNotResolved({ resourceName: name }); |
| 311 | } |
| 312 | |
| 313 | const observedLabels = tagRecord(current.labels); |
| 314 | const { upsert, removed } = diffLabels(observedLabels, desiredLabels); |
| 315 | const labelsChanged = upsert.length > 0 || removed.length > 0; |
| 316 | const descriptionChanged = (current.description ?? "") !== (news.description ?? ""); |
| 317 | const displayNameChanged = (current.displayName ?? "") !== (news.displayName ?? ""); |
| 318 | const executionChanged = |
| 319 | fingerprint(current.executionSpec) !== fingerprint(news.executionSpec); |
| 320 | const qualityChanged = |
| 321 | fingerprint(current.dataQualitySpec) !== fingerprint(news.dataQualitySpec); |
| 322 | const profileChanged = |
| 323 | fingerprint(current.dataProfileSpec) !== fingerprint(news.dataProfileSpec); |
| 324 | const discoveryChanged = |
| 325 | fingerprint(current.dataDiscoverySpec) !== fingerprint(news.dataDiscoverySpec); |
| 326 | const documentationChanged = |
| 327 | fingerprint(current.dataDocumentationSpec) !== fingerprint(news.dataDocumentationSpec); |
| 328 | const unstructuredChanged = |
| 329 | fingerprint(current.unstructuredDataProfileSpec) !== |
| 330 | fingerprint(news.unstructuredDataProfileSpec); |
| 331 | |
| 332 | if ( |
| 333 | labelsChanged || |
| 334 | descriptionChanged || |
| 335 | displayNameChanged || |
| 336 | executionChanged || |
| 337 | qualityChanged || |
| 338 | profileChanged || |
| 339 | discoveryChanged || |
| 340 | documentationChanged || |
| 341 | unstructuredChanged |
| 342 | ) { |
| 343 | const operation = yield* retryQuota( |
| 344 | dataplex.patchProjectsLocationsDataScans({ |
| 345 | name: current.name ?? name, |
| 346 | updateMask: [ |
| 347 | labelsChanged ? "labels" : undefined, |
| 348 | descriptionChanged ? "description" : undefined, |
| 349 | displayNameChanged ? "displayName" : undefined, |
| 350 | executionChanged ? "executionSpec" : undefined, |
| 351 | qualityChanged ? "dataQualitySpec" : undefined, |
| 352 | profileChanged ? "dataProfileSpec" : undefined, |
| 353 | discoveryChanged ? "dataDiscoverySpec" : undefined, |
| 354 | documentationChanged ? "dataDocumentationSpec" : undefined, |
| 355 | unstructuredChanged ? "unstructuredDataProfileSpec" : undefined, |
| 356 | ] |
| 357 | .filter((field): field is string => field !== undefined) |
| 358 | .join(","), |
| 359 | body: { |
| 360 | name: current.name ?? name, |
| 361 | description: news.description, |
| 362 | displayName: news.displayName, |
| 363 | labels: desiredLabels, |
| 364 | executionSpec: news.executionSpec, |
| 365 | dataQualitySpec: news.dataQualitySpec, |
| 366 | dataProfileSpec: news.dataProfileSpec, |
| 367 | dataDiscoverySpec: news.dataDiscoverySpec, |
| 368 | dataDocumentationSpec: news.dataDocumentationSpec, |
| 369 | unstructuredDataProfileSpec: news.unstructuredDataProfileSpec, |
| 370 | }, |
| 371 | }), |
| 372 | ); |
| 373 | yield* waitForOperation(operation); |
| 374 | current = yield* waitUntilExists(getByName(current.name ?? name), current.name ?? name); |
| 375 | } |
| 376 | |
| 377 | return toAttrs(current, env.project); |
| 378 | }), |
| 379 | |
| 380 | delete: Effect.fn(function* ({ output }) { |
| 381 | const operation = yield* retryQuota( |
| 382 | dataplex.deleteProjectsLocationsDataScans({ |
| 383 | name: output.name, |
| 384 | force: true, |
| 385 | }), |
| 386 | ).pipe( |
| 387 | Effect.retry({ |
| 388 | while: (error) => error._tag === "Conflict", |
| 389 | times: 8, |
| 390 | schedule: Schedule.spaced("2 seconds"), |
| 391 | }), |
| 392 | Effect.catchTag("NotFound", () => Effect.undefined), |
| 393 | ); |
| 394 | if (operation !== undefined) { |
| 395 | yield* waitForOperation(operation, { notFoundOk: true }); |
| 396 | } |
| 397 | yield* waitUntilGone(getByName(output.name), output.name); |
| 398 | }), |
| 399 | }); |