The best way to control Claude Code, Codex and more from any computer or phone

main
ts 506 lines 17.7 KB
Raw
1 import * as bigquery from "@distilled.cloud/gcp/bigquery_v2";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import { createPhysicalName } from "alchemy/PhysicalName";
5 import * as Provider from "alchemy/Provider";
6 import { Resource } from "alchemy/Resource";
7 import { tagRecord } from "alchemy/Tags";
8 import * as Data from "effect/Data";
9 import * as Effect from "effect/Effect";
10 import * as Stream from "effect/Stream";
11 import { GcpEnvironment } from "../Environment.ts";
12 import {
13 createInternalLabels,
14 diffLabels,
15 hasAlchemyLabels,
16 stripInternalLabels,
17 toLabels,
18 } from "../Labels.ts";
19 import type { Providers } from "../Providers.ts";
20
21 const MAX_DATASET_ID_LENGTH = 1024;
22
23 export type DatasetAccess = bigquery.DatasetAccessItem;
24
25 export type DatasetEncryption = {
26 /**
27 * Cloud KMS key used as the default encryption key for new tables,
28 * as `projects/{project}/locations/{location}/keyRings/{keyRing}/cryptoKeys/{cryptoKey}`.
29 */
30 kmsKeyName?: string;
31 };
32
33 export type DatasetProps = {
34 /**
35 * Dataset id (the `{datasetId}` segment of
36 * `projects/{project}/datasets/{datasetId}`). If omitted, a unique name
37 * is generated from the stack, stage, and logical id. Must contain only
38 * letters, numbers, and underscores; max 1024 characters. Immutable —
39 * changing it replaces the dataset.
40 */
41 datasetId?: string;
42 /**
43 * Geographic location (`US`, `EU`, `US-CENTRAL1`, `us-central1`, …).
44 * Immutable — changing it replaces the dataset.
45 * @default the stack's GCP region (`GCP.Region`, profile region, `us-central1`)
46 */
47 location?: string;
48 /**
49 * User labels. Alchemy ownership labels are merged in automatically.
50 */
51 labels?: Record<string, string>;
52 /**
53 * User-friendly description of the dataset.
54 */
55 description?: string;
56 /**
57 * Descriptive name shown in the console.
58 */
59 friendlyName?: string;
60 /**
61 * Default lifetime of newly created tables, in milliseconds. Minimum
62 * `"3600000"` (one hour). Set to `"0"` to clear.
63 */
64 defaultTableExpirationMs?: string;
65 /**
66 * Default partition expiration for new partitioned tables, in
67 * milliseconds. Set to `"0"` to clear.
68 */
69 defaultPartitionExpirationMs?: string;
70 /**
71 * Time travel window in hours (`"48"` to `"168"`). Defaults to `"168"`
72 * when omitted at create.
73 */
74 maxTimeTravelHours?: string;
75 /**
76 * Storage billing model (`LOGICAL` or `PHYSICAL`).
77 */
78 storageBillingModel?: bigquery.DatasetStorageBillingModelEnum | (string & {});
79 /**
80 * When true, dataset and table names are case-insensitive.
81 * @default false
82 */
83 isCaseInsensitive?: boolean;
84 /**
85 * Default collation for new tables (`"und:ci"` or `""`).
86 */
87 defaultCollation?: string;
88 /**
89 * Default rounding mode for new tables
90 * (`ROUND_HALF_AWAY_FROM_ZERO`, `ROUND_HALF_EVEN`).
91 */
92 defaultRoundingMode?: bigquery.DatasetDefaultRoundingModeEnum | (string & {});
93 /**
94 * Default Cloud KMS encryption for newly created tables.
95 */
96 defaultEncryptionConfiguration?: DatasetEncryption;
97 /**
98 * Access control entries. If omitted, BigQuery installs its default
99 * project-owner/reader/writer grants. Providing this list replaces the
100 * entire ACL.
101 */
102 access?: DatasetAccess[];
103 /**
104 * Delete all tables, views, and routines before destroying the dataset.
105 * @default false
106 */
107 forceDestroy?: boolean;
108 };
109
110 export type Dataset = Resource<
111 "GCP.BigQuery.Dataset",
112 DatasetProps,
113 {
114 /** Resource name `projects/{project}/datasets/{datasetId}`. */
115 name: string;
116 /** Dataset id. */
117 datasetId: string;
118 /** Project id. */
119 project: string;
120 /** Fully-qualified id `projectId:datasetId`. */
121 id: string;
122 /** Geographic location. */
123 location: string;
124 /** User labels (Alchemy ownership labels stripped). */
125 labels: Record<string, string>;
126 /** User-friendly description. */
127 description: string | undefined;
128 /** Descriptive console name. */
129 friendlyName: string | undefined;
130 /** Default table expiration, in milliseconds. */
131 defaultTableExpirationMs: string | undefined;
132 /** Default partition expiration, in milliseconds. */
133 defaultPartitionExpirationMs: string | undefined;
134 /** Time travel window in hours. */
135 maxTimeTravelHours: string | undefined;
136 /** Storage billing model. */
137 storageBillingModel: string | undefined;
138 /** Whether names are case-insensitive. */
139 isCaseInsensitive: boolean;
140 /** Default collation for new tables. */
141 defaultCollation: string | undefined;
142 /** Default rounding mode for new tables. */
143 defaultRoundingMode: string | undefined;
144 /** Default KMS key for new tables, if any. */
145 kmsKeyName: string | undefined;
146 /** Access control entries currently on the dataset. */
147 access: DatasetAccess[];
148 /** Dataset type (`DEFAULT`, `LINKED`, `EXTERNAL`, …). */
149 type: string | undefined;
150 /** Self-link URL. */
151 selfLink: string | undefined;
152 /** Server etag. */
153 etag: string | undefined;
154 /** Creation time in milliseconds since epoch. */
155 creationTime: string | undefined;
156 /** Last-modified time in milliseconds since epoch. */
157 lastModifiedTime: string | undefined;
158 },
159 never,
160 Providers
161 >;
162
163 /**
164 * A Google BigQuery dataset — a container for tables, views, and routines.
165 *
166 * Changing `datasetId` or `location` replaces the dataset. Labels,
167 * description, expiration, collation, encryption, and access update in
168 * place. Destroy fails if the dataset still contains tables unless
169 * `forceDestroy` is set.
170 *
171 * ### Creating a Dataset
172 * **Example:** Generated name
173 * ```typescript
174 * const dataset = yield* GCP.BigQuery.Dataset("Analytics", {});
175 * ```
176 *
177 * **Example:** Explicit id, location, and labels
178 * ```typescript
179 * const dataset = yield* GCP.BigQuery.Dataset("Analytics", {
180 * datasetId: "app_analytics",
181 * location: "US-CENTRAL1",
182 * labels: { env: "prod" },
183 * description: "application events",
184 * forceDestroy: true,
185 * });
186 * ```
187 *
188 * ### Querying
189 * **Example:** Run a GoogleSQL query against the dataset
190 * ```typescript
191 * const query = yield* GCP.BigQuery.Query(dataset);
192 * const result = yield* query({ query: "SELECT 1 AS n" });
193 * ```
194 *
195 * @resource
196 * @category BigQuery
197 */
198 export const Dataset = Resource<Dataset>("GCP.BigQuery.Dataset");
199
200 export class DatasetNotResolved extends Data.TaggedError("GCP.BigQuery.DatasetNotResolved")<{
201 resourceName: string;
202 }> {}
203
204 const userLabels = (
205 labels: Record<string, string | undefined> | null | undefined,
206 ): Record<string, string> => stripInternalLabels(tagRecord(labels));
207
208 const resourceName = (project: string, datasetId: string) =>
209 `projects/${project}/datasets/${datasetId}`;
210
211 const datasetIdOf = (dataset: bigquery.Dataset | bigquery.DatasetListDatasetsItem) => {
212 const fromRef = dataset.datasetReference?.datasetId;
213 if (fromRef !== undefined && fromRef.length > 0) return fromRef;
214 const id = dataset.id ?? "";
215 const colon = id.indexOf(":");
216 return colon >= 0 ? id.slice(colon + 1) : id;
217 };
218
219 const projectOf = (
220 dataset: bigquery.Dataset | bigquery.DatasetListDatasetsItem,
221 fallback: string,
222 ) => {
223 const fromRef = dataset.datasetReference?.projectId;
224 if (fromRef !== undefined && fromRef.length > 0) return fromRef;
225 const id = dataset.id ?? "";
226 const colon = id.indexOf(":");
227 return colon > 0 ? id.slice(0, colon) : fallback;
228 };
229
230 const toId = (id: string, datasetId: string | undefined, existing?: string) =>
231 Effect.gen(function* () {
232 if (datasetId !== undefined) return datasetId;
233 if (existing !== undefined) return existing;
234 const generated = yield* createPhysicalName({
235 id,
236 maxLength: MAX_DATASET_ID_LENGTH,
237 lowercase: true,
238 delimiter: "_",
239 });
240 return generated.replaceAll("-", "_");
241 });
242
243 const toAttrs = (
244 dataset: bigquery.Dataset | bigquery.DatasetListDatasetsItem,
245 fallbackProject: string,
246 region: string,
247 ) => {
248 const datasetId = datasetIdOf(dataset);
249 const project = projectOf(dataset, fallbackProject);
250 const full = dataset as bigquery.Dataset;
251 return {
252 name: resourceName(project, datasetId),
253 datasetId,
254 project,
255 id: full.id ?? `${project}:${datasetId}`,
256 location: dataset.location ?? region,
257 labels: userLabels(dataset.labels),
258 description: full.description,
259 friendlyName: dataset.friendlyName,
260 defaultTableExpirationMs: full.defaultTableExpirationMs,
261 defaultPartitionExpirationMs: full.defaultPartitionExpirationMs,
262 maxTimeTravelHours: full.maxTimeTravelHours,
263 storageBillingModel: full.storageBillingModel,
264 isCaseInsensitive: full.isCaseInsensitive === true,
265 defaultCollation: full.defaultCollation,
266 defaultRoundingMode: full.defaultRoundingMode,
267 kmsKeyName: full.defaultEncryptionConfiguration?.kmsKeyName,
268 access: full.access ?? [],
269 type: dataset.type,
270 selfLink: full.selfLink,
271 etag: full.etag,
272 creationTime: full.creationTime,
273 lastModifiedTime: full.lastModifiedTime,
274 };
275 };
276
277 const getById = (project: string, datasetId: string) =>
278 bigquery
279 .getDatasets({ projectId: project, datasetId })
280 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
281
282 const sameString = (left: string | undefined, right: string | undefined) =>
283 (left ?? "") === (right ?? "");
284
285 const accessJson = (access: readonly DatasetAccess[] | undefined) => JSON.stringify(access ?? []);
286
287 export const DatasetProvider = () =>
288 Provider.succeed(Dataset, {
289 stables: ["name", "datasetId", "project", "id", "location", "creationTime"],
290
291 diff: Effect.fn(function* ({ news, olds, output }) {
292 if (!isResolved(news)) return undefined;
293 const previousId = olds?.datasetId ?? output?.datasetId;
294 if (
295 previousId !== undefined &&
296 news.datasetId !== undefined &&
297 news.datasetId !== previousId
298 ) {
299 return { action: "replace" as const, deleteFirst: false };
300 }
301 const previousLocation = olds?.location ?? output?.location;
302 const nextLocation = news.location;
303 if (
304 previousLocation !== undefined &&
305 nextLocation !== undefined &&
306 previousLocation.toUpperCase() !== nextLocation.toUpperCase()
307 ) {
308 const nextId = news.datasetId ?? previousId;
309 return {
310 action: "replace" as const,
311 deleteFirst: nextId !== undefined && nextId === previousId,
312 };
313 }
314 return undefined;
315 }),
316
317 read: Effect.fn(function* ({ id, olds, output }) {
318 const env = yield* GcpEnvironment.current;
319 const datasetId = yield* toId(id, olds?.datasetId, output?.datasetId);
320 const project = output?.project ?? env.project;
321 const existing = yield* getById(project, datasetId);
322 if (existing === undefined) return undefined;
323 const attrs = toAttrs(existing, project, env.region);
324 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
325 }),
326
327 list: () =>
328 Effect.gen(function* () {
329 const env = yield* GcpEnvironment.current;
330 return yield* bigquery.listDatasets
331 .pages({
332 projectId: env.project,
333 maxResults: 1000,
334 all: true,
335 })
336 .pipe(
337 Stream.flatMap((page) => Stream.fromIterable(page.datasets ?? [])),
338 Stream.filter((dataset) =>
339 Object.keys(dataset.labels ?? {}).some((key) => key.startsWith("alchemy-")),
340 ),
341 Stream.map((dataset) => toAttrs(dataset, env.project, env.region)),
342 Stream.runCollect,
343 Effect.map((chunk) => Array.from(chunk)),
344 );
345 }),
346
347 reconcile: Effect.fn(function* ({ id, news, output }) {
348 const env = yield* GcpEnvironment.current;
349 const datasetId = yield* toId(id, news.datasetId, output?.datasetId);
350 const project = output?.project ?? env.project;
351 const location = news.location ?? output?.location ?? env.region;
352 const desiredLabels = {
353 ...toLabels(news.labels),
354 ...(yield* createInternalLabels(id)),
355 };
356 const name = resourceName(project, datasetId);
357
358 let current = yield* getById(project, datasetId);
359
360 if (current === undefined) {
361 const created = yield* bigquery
362 .insertDatasets({
363 projectId: project,
364 body: {
365 datasetReference: { projectId: project, datasetId },
366 location,
367 labels: desiredLabels,
368 description: news.description,
369 friendlyName: news.friendlyName,
370 defaultTableExpirationMs: news.defaultTableExpirationMs,
371 defaultPartitionExpirationMs: news.defaultPartitionExpirationMs,
372 maxTimeTravelHours: news.maxTimeTravelHours,
373 storageBillingModel: news.storageBillingModel,
374 isCaseInsensitive: news.isCaseInsensitive === true ? true : undefined,
375 defaultCollation: news.defaultCollation,
376 defaultRoundingMode: news.defaultRoundingMode,
377 defaultEncryptionConfiguration: news.defaultEncryptionConfiguration,
378 access: news.access,
379 },
380 })
381 .pipe(Effect.catchTag("Conflict", () => getById(project, datasetId)));
382 current = created ?? undefined;
383 }
384
385 if (current === undefined) {
386 return yield* new DatasetNotResolved({ resourceName: name });
387 }
388
389 const observedLabels = tagRecord(current.labels);
390 const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
391 if (removed.length > 0) {
392 // PATCH merges labels; removing one needs a JSON null the typed body
393 // cannot carry, so replace the labels with PUT (get-modify-put).
394 current = yield* bigquery.updateDatasets({
395 projectId: project,
396 datasetId,
397 body: { ...current, labels: desiredLabels },
398 });
399 }
400 const labelsChanged = removed.length === 0 && upsert.length > 0;
401 const descriptionChanged = !sameString(current.description, news.description);
402 const friendlyNameChanged = !sameString(current.friendlyName, news.friendlyName);
403 const tableExpirationChanged =
404 news.defaultTableExpirationMs !== undefined &&
405 !sameString(current.defaultTableExpirationMs, news.defaultTableExpirationMs);
406 const partitionExpirationChanged =
407 news.defaultPartitionExpirationMs !== undefined &&
408 !sameString(current.defaultPartitionExpirationMs, news.defaultPartitionExpirationMs);
409 const timeTravelChanged =
410 news.maxTimeTravelHours !== undefined &&
411 !sameString(current.maxTimeTravelHours, news.maxTimeTravelHours);
412 const billingChanged =
413 news.storageBillingModel !== undefined &&
414 (current.storageBillingModel ?? "") !== news.storageBillingModel;
415 const caseChanged =
416 (current.isCaseInsensitive === true) !== (news.isCaseInsensitive === true);
417 const collationChanged =
418 news.defaultCollation !== undefined &&
419 !sameString(current.defaultCollation, news.defaultCollation);
420 const roundingChanged =
421 news.defaultRoundingMode !== undefined &&
422 (current.defaultRoundingMode ?? "") !== news.defaultRoundingMode;
423 const encryptionChanged =
424 news.defaultEncryptionConfiguration !== undefined &&
425 (current.defaultEncryptionConfiguration?.kmsKeyName ?? "") !==
426 (news.defaultEncryptionConfiguration.kmsKeyName ?? "");
427 const accessChanged =
428 news.access !== undefined && accessJson(current.access) !== accessJson(news.access);
429
430 const metadataChanged =
431 labelsChanged ||
432 descriptionChanged ||
433 friendlyNameChanged ||
434 tableExpirationChanged ||
435 partitionExpirationChanged ||
436 timeTravelChanged ||
437 billingChanged ||
438 caseChanged ||
439 collationChanged ||
440 roundingChanged ||
441 encryptionChanged;
442
443 if (metadataChanged || accessChanged) {
444 const body: bigquery.Dataset = {};
445 if (labelsChanged) {
446 body.labels = desiredLabels;
447 }
448 if (descriptionChanged) {
449 body.description = news.description ?? "";
450 }
451 if (friendlyNameChanged) {
452 body.friendlyName = news.friendlyName ?? "";
453 }
454 if (tableExpirationChanged) {
455 body.defaultTableExpirationMs = news.defaultTableExpirationMs;
456 }
457 if (partitionExpirationChanged) {
458 body.defaultPartitionExpirationMs = news.defaultPartitionExpirationMs;
459 }
460 if (timeTravelChanged) {
461 body.maxTimeTravelHours = news.maxTimeTravelHours;
462 }
463 if (billingChanged) {
464 body.storageBillingModel = news.storageBillingModel;
465 }
466 if (caseChanged) {
467 body.isCaseInsensitive = news.isCaseInsensitive === true;
468 }
469 if (collationChanged) {
470 body.defaultCollation = news.defaultCollation;
471 }
472 if (roundingChanged) {
473 body.defaultRoundingMode = news.defaultRoundingMode;
474 }
475 if (encryptionChanged) {
476 body.defaultEncryptionConfiguration = news.defaultEncryptionConfiguration;
477 }
478 if (accessChanged) {
479 body.access = news.access;
480 }
481 current = yield* bigquery.patchDatasets({
482 projectId: project,
483 datasetId,
484 updateMode:
485 accessChanged && metadataChanged
486 ? "UPDATE_FULL"
487 : accessChanged
488 ? "UPDATE_ACL"
489 : "UPDATE_METADATA",
490 body,
491 });
492 }
493
494 return toAttrs(current, project, env.region);
495 }),
496
497 delete: Effect.fn(function* ({ olds, output, force }) {
498 yield* bigquery
499 .deleteDatasets({
500 projectId: output.project,
501 datasetId: output.datasetId,
502 deleteContents: olds.forceDestroy === true || force === true,
503 })
504 .pipe(Effect.catchTag("NotFound", () => Effect.void));
505 }),
506 });