@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 856 lines 28.6 KB
Raw
1 import * as bigquery from "@distilled.cloud/gcp/bigquery_v2";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import { createPhysicalName } from "alchemy/PhysicalName";
5 import * as Provider from "alchemy/Provider";
6 import { Resource } from "alchemy/Resource";
7 import { tagRecord } from "alchemy/Tags";
8 import * as Data from "effect/Data";
9 import * as Effect from "effect/Effect";
10 import * as Stream from "effect/Stream";
11 import { GcpEnvironment } from "../Environment.ts";
12 import {
13 createInternalLabels,
14 diffLabels,
15 hasAlchemyLabels,
16 stripInternalLabels,
17 toLabels,
18 } from "../Labels.ts";
19 import type { Providers } from "../Providers.ts";
20
21 const MAX_NAME_LENGTH = 1024;
22 const DEFAULT_TYPE = "TABLE";
23
24 const TYPE_ALIASES: Record<string, string> = {
25 INT64: "INTEGER",
26 FLOAT64: "FLOAT",
27 BOOL: "BOOLEAN",
28 STRUCT: "RECORD",
29 };
30
31 export type TableField = {
32 /** Field name. Letters, numbers, underscores; max 300 characters. */
33 name: string;
34 /**
35 * Field type (`STRING`, `INTEGER`/`INT64`, `FLOAT`/`FLOAT64`,
36 * `BOOLEAN`/`BOOL`, `TIMESTAMP`, `DATE`, `RECORD`/`STRUCT`, …).
37 */
38 type: string;
39 /**
40 * Field mode (`NULLABLE`, `REQUIRED`, `REPEATED`).
41 * @default "NULLABLE"
42 */
43 mode?: string;
44 /** Field description. Max 1,024 characters. */
45 description?: string;
46 /** Nested fields when `type` is `RECORD` / `STRUCT`. */
47 fields?: TableField[];
48 /** Max UTF-8 length (`STRING`) or byte length (`BYTES`). */
49 maxLength?: string;
50 /** Precision for `NUMERIC` / `BIGNUMERIC`. */
51 precision?: string;
52 /** Scale for `NUMERIC` / `BIGNUMERIC`. */
53 scale?: string;
54 /** SQL expression for the column default. */
55 defaultValueExpression?: string;
56 /** Collation for `STRING` fields (`und:ci` or `""`). */
57 collation?: string;
58 };
59
60 export type TableTimePartitioning = {
61 /**
62 * Partition granularity (`DAY`, `HOUR`, `MONTH`, `YEAR`). Immutable —
63 * changing it replaces the table.
64 */
65 type: string;
66 /**
67 * Partitioning column (`TIMESTAMP` or `DATE`). Omit to use
68 * `_PARTITIONTIME`. Immutable — changing it replaces the table.
69 */
70 field?: string;
71 /** Partition expiration in milliseconds. */
72 expirationMs?: string;
73 };
74
75 export type TableClustering = {
76 /** Top-level clustering columns, most selective first. */
77 fields?: string[];
78 };
79
80 export type TableView = {
81 /** Query that defines the view. Updates in place. */
82 query: string;
83 /**
84 * Use legacy SQL instead of GoogleSQL. The BigQuery API defaults views
85 * to legacy SQL; this resource defaults them to GoogleSQL.
86 * @default false
87 */
88 useLegacySql?: boolean;
89 };
90
91 export type TableHivePartitioningOptions = {
92 /**
93 * How partition keys are read from the source paths: `AUTO` infers
94 * their types, `STRINGS` reads every key as `STRING`, and `CUSTOM`
95 * takes keys and types from `sourceUriPrefix`.
96 */
97 mode?: string;
98 /**
99 * Prefix shared by every source URI, up to the first partition key
100 * (`gs://bucket/events`). In `CUSTOM` mode it also declares the keys:
101 * `gs://bucket/events/{day:DATE}`.
102 */
103 sourceUriPrefix?: string;
104 /**
105 * Require a filter on a partition key in every query of the table.
106 * @default false
107 */
108 requirePartitionFilter?: boolean;
109 };
110
111 export type TableExternalDataConfiguration = {
112 /** Source URIs, e.g. `gs://bucket/events/*`. */
113 sourceUris: string[];
114 /**
115 * Source format: `PARQUET`, `NEWLINE_DELIMITED_JSON`, `CSV`, `AVRO`,
116 * `ORC`, …
117 */
118 sourceFormat: string;
119 /**
120 * Infer the schema from the data. Self-describing formats such as
121 * Parquet do not need it.
122 */
123 autodetect?: boolean;
124 /** Hive partitioning of the source paths. */
125 hivePartitioningOptions?: TableHivePartitioningOptions;
126 };
127
128 export type TableProps = {
129 /**
130 * Dataset id (the `{dataset}` segment of
131 * `projects/{project}/datasets/{dataset}`) or a full dataset resource
132 * name. Immutable — changing it replaces the table.
133 */
134 datasetId: string;
135 /**
136 * Table id (the `{table}` segment of
137 * `projects/{project}/datasets/{dataset}/tables/{table}`). If omitted,
138 * a unique name is generated from the stack, stage, and logical id.
139 * Must contain only letters, numbers, and underscores; max 1,024
140 * characters. Immutable — changing it replaces the table.
141 */
142 tableId?: string;
143 /**
144 * User labels. Alchemy ownership labels are merged in automatically.
145 */
146 labels?: Record<string, string>;
147 /**
148 * Table schema. Adding columns updates in place; removing or changing
149 * the type of an existing column is rejected by BigQuery.
150 */
151 schema?: TableField[];
152 /** User-friendly description of this table. */
153 description?: string;
154 /** Descriptive display name. */
155 friendlyName?: string;
156 /**
157 * Expiration time as milliseconds since the Unix epoch (string). Omit
158 * for no expiration.
159 */
160 expirationTime?: string;
161 /**
162 * Time-based partitioning. `type` and `field` are immutable.
163 */
164 timePartitioning?: TableTimePartitioning;
165 /**
166 * Clustering columns. Usually combined with time partitioning.
167 */
168 clustering?: TableClustering;
169 /**
170 * Require a partition filter in queries that can be used for partition
171 * elimination.
172 * @default false
173 */
174 requirePartitionFilter?: boolean;
175 /**
176 * Cloud KMS key used to encrypt table data, as
177 * `projects/{project}/locations/{location}/keyRings/{keyRing}/cryptoKeys/{cryptoKey}`.
178 * Immutable — changing it replaces the table.
179 */
180 kmsKeyName?: string;
181 /**
182 * Make the table a logical view defined by a query. Mutually exclusive
183 * with `externalDataConfiguration`. Switching between a native table, a
184 * view, and an external table replaces the table.
185 */
186 view?: TableView;
187 /**
188 * Make the table an external table over data outside BigQuery, such as
189 * files in Cloud Storage. Mutually exclusive with `view`. Updates in
190 * place; switching to or from a native table or a view replaces it.
191 */
192 externalDataConfiguration?: TableExternalDataConfiguration;
193 };
194
195 export type Table = Resource<
196 "GCP.BigQuery.Table",
197 TableProps,
198 {
199 /**
200 * Resource path `projects/{project}/datasets/{dataset}/tables/{table}`.
201 */
202 name: string;
203 /** Opaque id `project:dataset.table`. */
204 id: string;
205 /** Table id (last path segment). */
206 tableId: string;
207 /** Dataset id. */
208 datasetId: string;
209 /** Project id. */
210 project: string;
211 /** Table type (`TABLE`, `VIEW`, `EXTERNAL`, …). */
212 type: string;
213 /** User labels (Alchemy ownership labels stripped). */
214 labels: Record<string, string>;
215 /** Schema fields, if set. */
216 schema: TableField[] | undefined;
217 /** Description, if set. */
218 description: string | undefined;
219 /** Friendly name, if set. */
220 friendlyName: string | undefined;
221 /** Expiration time in milliseconds since epoch, if set. */
222 expirationTime: string | undefined;
223 /** Dataset location (`US`, `us-central1`, …). */
224 location: string | undefined;
225 /** Time partitioning, if configured. */
226 timePartitioning: TableTimePartitioning | undefined;
227 /** Clustering, if configured. */
228 clustering: TableClustering | undefined;
229 /** Whether queries must specify a partition filter. */
230 requirePartitionFilter: boolean;
231 /** CMEK key, if set. */
232 kmsKeyName: string | undefined;
233 /** View definition, for views. */
234 view: TableView | undefined;
235 /** External data configuration, for external tables. */
236 externalDataConfiguration: TableExternalDataConfiguration | undefined;
237 /** Creation time in milliseconds since epoch. */
238 creationTime: string | undefined;
239 /** Row count, excluding the streaming buffer. */
240 numRows: string | undefined;
241 /** Self-link URL. */
242 selfLink: string | undefined;
243 },
244 never,
245 Providers
246 >;
247
248 /**
249 * A Google BigQuery table: a native table, a view, or an external table.
250 *
251 * Materialized views and snapshots are not managed by this resource.
252 * `datasetId`, `tableId`, time partitioning `type`/`field`, `kmsKeyName`,
253 * and the kind of table (native, `view`, or `externalDataConfiguration`)
254 * are immutable (changing them replaces the table). Schema, labels,
255 * description, clustering, partition expiration, the view query, and the
256 * external data configuration update in place.
257 *
258 * ### Creating a Table
259 * **Example:** Generated name in an existing dataset
260 * ```typescript
261 * const events = yield* GCP.BigQuery.Table("Events", {
262 * datasetId: "analytics",
263 * schema: [
264 * { name: "id", type: "STRING" },
265 * { name: "created_at", type: "TIMESTAMP" },
266 * ],
267 * });
268 * ```
269 *
270 * **Example:** Explicit id, labels, and partitioning
271 * ```typescript
272 * const events = yield* GCP.BigQuery.Table("Events", {
273 * datasetId: "analytics",
274 * tableId: "order_events",
275 * labels: { env: "prod" },
276 * schema: [
277 * { name: "id", type: "STRING" },
278 * { name: "created_at", type: "TIMESTAMP" },
279 * ],
280 * timePartitioning: { type: "DAY", field: "created_at" },
281 * clustering: { fields: ["id"] },
282 * });
283 * ```
284 *
285 * ### Views and External Tables
286 * **Example:** View over another table
287 * ```typescript
288 * const daily = yield* GCP.BigQuery.Table("DailyEvents", {
289 * datasetId: "analytics",
290 * view: {
291 * query: "SELECT DATE(created_at) AS day, COUNT(*) AS n FROM analytics.order_events GROUP BY day",
292 * },
293 * });
294 * ```
295 *
296 * **Example:** Hive-partitioned Parquet files in Cloud Storage
297 * ```typescript
298 * const inventory = yield* GCP.BigQuery.Table("Inventory", {
299 * datasetId: "ops",
300 * externalDataConfiguration: {
301 * sourceUris: ["gs://my-bucket/inventory/*.parquet"],
302 * sourceFormat: "PARQUET",
303 * hivePartitioningOptions: {
304 * mode: "CUSTOM",
305 * sourceUriPrefix: "gs://my-bucket/inventory/{bucket:STRING}/{date:DATE}",
306 * },
307 * },
308 * });
309 * ```
310 *
311 * ### Inserting Rows
312 * **Example:** Stream rows with InsertAll
313 * ```typescript
314 * const insertAll = yield* GCP.BigQuery.InsertAll(events);
315 * yield* insertAll({
316 * body: { rows: [{ json: { id: "1", created_at: "2024-01-01T00:00:00Z" } }] },
317 * });
318 * ```
319 *
320 * @resource
321 * @category BigQuery
322 */
323 export const Table = Resource<Table>("GCP.BigQuery.Table");
324
325 export class TableNotResolved extends Data.TaggedError("GCP.BigQuery.TableNotResolved")<{
326 resourceName: string;
327 }> {}
328
329 /** `view` and `externalDataConfiguration` were both set; a table is one kind. */
330 export class ConflictingTableKind extends Data.TaggedError("GCP.BigQuery.ConflictingTableKind")<{
331 message: string;
332 }> {}
333
334 const lastSegment = (value: string) => {
335 const trimmed = value.replace(/\/+$/, "");
336 const parts = trimmed.split("/");
337 return parts[parts.length - 1] || trimmed;
338 };
339
340 const datasetIdOf = (datasetId: string) =>
341 datasetId.includes("/") ? lastSegment(datasetId) : datasetId;
342
343 const resourceName = (project: string, datasetId: string, tableId: string) =>
344 `projects/${project}/datasets/${datasetId}/tables/${tableId}`;
345
346 const userLabels = (
347 labels: Record<string, string | undefined> | null | undefined,
348 ): Record<string, string> => stripInternalLabels(tagRecord(labels));
349
350 const toId = (id: string, tableId: string | undefined, existing?: string) =>
351 Effect.gen(function* () {
352 if (tableId !== undefined) return tableId;
353 if (existing !== undefined) return existing;
354 const generated = yield* createPhysicalName({
355 id,
356 maxLength: MAX_NAME_LENGTH,
357 lowercase: true,
358 delimiter: "_",
359 });
360 return generated.replaceAll("-", "_");
361 });
362
363 const toFieldSchema = (fields: TableField[]): bigquery.TableFieldSchema[] =>
364 fields.map((field) => ({
365 name: field.name,
366 type: field.type,
367 mode: field.mode,
368 description: field.description,
369 fields: field.fields ? toFieldSchema(field.fields) : undefined,
370 maxLength: field.maxLength,
371 precision: field.precision,
372 scale: field.scale,
373 defaultValueExpression: field.defaultValueExpression,
374 collation: field.collation,
375 }));
376
377 const schemaOf = (schema: bigquery.TableSchema | undefined): TableField[] | undefined => {
378 const fields = schema?.fields;
379 if (fields === undefined || fields.length === 0) return undefined;
380 return fields.flatMap((field) =>
381 field.name && field.type
382 ? [
383 {
384 name: field.name,
385 type: field.type,
386 mode: field.mode,
387 description: field.description,
388 fields: schemaOf({ fields: field.fields }),
389 maxLength: field.maxLength,
390 precision: field.precision,
391 scale: field.scale,
392 defaultValueExpression: field.defaultValueExpression,
393 collation: field.collation,
394 },
395 ]
396 : [],
397 );
398 };
399
400 const normalizeType = (type: string) => {
401 const upper = type.toUpperCase();
402 return TYPE_ALIASES[upper] ?? upper;
403 };
404
405 const canonField = (field: TableField): unknown => ({
406 name: field.name,
407 type: normalizeType(field.type),
408 mode: (field.mode ?? "NULLABLE").toUpperCase(),
409 description: field.description ?? "",
410 maxLength: field.maxLength ?? "",
411 precision: field.precision ?? "",
412 scale: field.scale ?? "",
413 defaultValueExpression: field.defaultValueExpression ?? "",
414 collation: field.collation ?? "",
415 fields: (field.fields ?? []).map(canonField),
416 });
417
418 const jsonEqual = (left: unknown, right: unknown) =>
419 JSON.stringify(left ?? null) === JSON.stringify(right ?? null);
420
421 const schemaChanged = (
422 desired: TableField[] | undefined,
423 observed: bigquery.TableSchema | undefined,
424 ) =>
425 desired !== undefined &&
426 !jsonEqual(desired.map(canonField), (schemaOf(observed) ?? []).map(canonField));
427
428 const timePartitioningOf = (
429 value: bigquery.TimePartitioning | undefined,
430 ): TableTimePartitioning | undefined => {
431 if (value?.type === undefined) return undefined;
432 return {
433 type: value.type,
434 field: value.field,
435 expirationMs: value.expirationMs,
436 };
437 };
438
439 const clusteringOf = (value: bigquery.Clustering | undefined): TableClustering | undefined => {
440 const fields = (value?.fields ?? []).filter(
441 (field): field is string => field !== undefined && field.length > 0,
442 );
443 return fields.length > 0 ? { fields } : undefined;
444 };
445
446 const sameOptionalString = (left: string | undefined, right: string | undefined) =>
447 (left ?? "") === (right ?? "");
448
449 /** Table kind as BigQuery reports it in `type`. */
450 const kindOf = (props: Pick<TableProps, "view" | "externalDataConfiguration">) =>
451 props.view !== undefined
452 ? "VIEW"
453 : props.externalDataConfiguration !== undefined
454 ? "EXTERNAL"
455 : DEFAULT_TYPE;
456
457 // The API defaults views to legacy SQL, so the flag is always sent.
458 const toView = (view: TableView): bigquery.ViewDefinition => ({
459 query: view.query,
460 useLegacySql: view.useLegacySql ?? false,
461 });
462
463 const viewOf = (value: bigquery.ViewDefinition | undefined): TableView | undefined =>
464 value?.query === undefined
465 ? undefined
466 : { query: value.query, useLegacySql: value.useLegacySql === true };
467
468 const viewChanged = (
469 desired: TableView | undefined,
470 observed: bigquery.ViewDefinition | undefined,
471 ) =>
472 desired !== undefined &&
473 (desired.query !== observed?.query ||
474 (desired.useLegacySql ?? false) !== (observed?.useLegacySql === true));
475
476 const toExternal = (
477 config: TableExternalDataConfiguration,
478 ): bigquery.ExternalDataConfiguration => ({
479 sourceUris: config.sourceUris,
480 sourceFormat: config.sourceFormat,
481 autodetect: config.autodetect,
482 hivePartitioningOptions: config.hivePartitioningOptions,
483 });
484
485 const externalOf = (
486 value: bigquery.ExternalDataConfiguration | undefined,
487 ): TableExternalDataConfiguration | undefined => {
488 if (value?.sourceFormat === undefined) return undefined;
489 const hive = value.hivePartitioningOptions;
490 return {
491 sourceUris: [...(value.sourceUris ?? [])],
492 sourceFormat: value.sourceFormat,
493 autodetect: value.autodetect,
494 hivePartitioningOptions:
495 hive === undefined
496 ? undefined
497 : {
498 mode: hive.mode,
499 sourceUriPrefix: hive.sourceUriPrefix,
500 requirePartitionFilter: hive.requirePartitionFilter === true,
501 },
502 };
503 };
504
505 // Only the keys the user set are compared: BigQuery fills in defaults.
506 const differs = <T>(desired: T | undefined, observed: T | undefined) =>
507 desired !== undefined && desired !== observed;
508
509 const externalChanged = (
510 desired: TableExternalDataConfiguration | undefined,
511 observed: bigquery.ExternalDataConfiguration | undefined,
512 ) => {
513 if (desired === undefined) return false;
514 if (observed === undefined) return true;
515 const hive = desired.hivePartitioningOptions;
516 const observedHive = observed.hivePartitioningOptions;
517 return (
518 !jsonEqual(desired.sourceUris, observed.sourceUris ?? []) ||
519 desired.sourceFormat.toUpperCase() !== (observed.sourceFormat ?? "").toUpperCase() ||
520 differs(desired.autodetect, observed.autodetect === true) ||
521 (hive !== undefined &&
522 (observedHive === undefined ||
523 differs(hive.mode, observedHive.mode) ||
524 differs(hive.sourceUriPrefix, observedHive.sourceUriPrefix) ||
525 differs(hive.requirePartitionFilter, observedHive.requirePartitionFilter === true)))
526 );
527 };
528
529 const toAttrs = (table: bigquery.Table | bigquery.TableListTablesItem, project: string) => {
530 const full = table as bigquery.Table;
531 const ref = table.tableReference;
532 const tableId = ref?.tableId ?? "";
533 const datasetId = ref?.datasetId ?? "";
534 const projectId = ref?.projectId ?? project;
535 return {
536 name: resourceName(projectId, datasetId, tableId),
537 id: full.id ?? `${projectId}:${datasetId}.${tableId}`,
538 tableId,
539 datasetId,
540 project: projectId,
541 type: table.type ?? DEFAULT_TYPE,
542 labels: userLabels(table.labels),
543 schema: schemaOf(full.schema),
544 description: full.description,
545 friendlyName: table.friendlyName,
546 expirationTime: table.expirationTime,
547 location: full.location,
548 timePartitioning: timePartitioningOf(table.timePartitioning),
549 clustering: clusteringOf(table.clustering),
550 requirePartitionFilter: table.requirePartitionFilter === true,
551 kmsKeyName: full.encryptionConfiguration?.kmsKeyName,
552 view: viewOf(full.view),
553 externalDataConfiguration: externalOf(full.externalDataConfiguration),
554 creationTime: table.creationTime,
555 numRows: full.numRows,
556 selfLink: full.selfLink,
557 };
558 };
559
560 const getByRef = (projectId: string, datasetId: string, tableId: string) =>
561 bigquery
562 .getTables({
563 projectId,
564 datasetId,
565 tableId,
566 view: "FULL",
567 })
568 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
569
570 const toTableBody = (
571 projectId: string,
572 datasetId: string,
573 tableId: string,
574 news: TableProps,
575 labels: Record<string, string>,
576 ): bigquery.Table => {
577 const body: bigquery.Table = {
578 tableReference: { projectId, datasetId, tableId },
579 labels,
580 };
581 if (news.schema !== undefined) {
582 body.schema = { fields: toFieldSchema(news.schema) };
583 }
584 if (news.description !== undefined) {
585 body.description = news.description;
586 }
587 if (news.friendlyName !== undefined) {
588 body.friendlyName = news.friendlyName;
589 }
590 if (news.expirationTime !== undefined) {
591 body.expirationTime = news.expirationTime;
592 }
593 if (news.timePartitioning !== undefined) {
594 body.timePartitioning = news.timePartitioning;
595 }
596 if (news.clustering !== undefined) {
597 body.clustering = news.clustering;
598 }
599 if (news.requirePartitionFilter !== undefined) {
600 body.requirePartitionFilter = news.requirePartitionFilter;
601 }
602 if (news.kmsKeyName !== undefined) {
603 body.encryptionConfiguration = { kmsKeyName: news.kmsKeyName };
604 }
605 if (news.view !== undefined) {
606 body.view = toView(news.view);
607 }
608 if (news.externalDataConfiguration !== undefined) {
609 body.externalDataConfiguration = toExternal(news.externalDataConfiguration);
610 }
611 return body;
612 };
613
614 export const TableProvider = () =>
615 Provider.succeed(Table, {
616 stables: [
617 "name",
618 "id",
619 "tableId",
620 "datasetId",
621 "project",
622 "location",
623 "creationTime",
624 "selfLink",
625 ],
626
627 diff: Effect.fn(function* ({ news, olds, output }) {
628 if (!isResolved(news)) return undefined;
629 const previousTableId = olds?.tableId ?? output?.tableId;
630 const nextTableId = news.tableId ?? previousTableId;
631 const tableIdChanged =
632 news.tableId !== undefined &&
633 previousTableId !== undefined &&
634 news.tableId !== previousTableId;
635
636 const previousDataset = olds?.datasetId ?? output?.datasetId;
637 const nextDataset = datasetIdOf(news.datasetId);
638 const datasetChanged =
639 previousDataset !== undefined && datasetIdOf(previousDataset) !== nextDataset;
640
641 const previousPartition = olds?.timePartitioning ?? output?.timePartitioning;
642 const nextPartition = news.timePartitioning;
643 const partitionChanged =
644 nextPartition !== undefined &&
645 ((previousPartition?.type ?? "").toUpperCase() !== nextPartition.type.toUpperCase() ||
646 (previousPartition?.field ?? "") !== (nextPartition.field ?? ""));
647
648 const previousKms = olds?.kmsKeyName ?? output?.kmsKeyName ?? "";
649 const nextKms = news.kmsKeyName ?? previousKms;
650 const kmsChanged = previousKms !== nextKms;
651
652 const previousKind = olds !== undefined ? kindOf(olds) : output?.type;
653 const kindChanged = previousKind !== undefined && previousKind !== kindOf(news);
654
655 if (!tableIdChanged && !datasetChanged && !partitionChanged && !kmsChanged && !kindChanged) {
656 return undefined;
657 }
658 return {
659 action: "replace" as const,
660 deleteFirst:
661 !tableIdChanged &&
662 !datasetChanged &&
663 nextTableId !== undefined &&
664 previousTableId !== undefined &&
665 nextTableId === previousTableId,
666 };
667 }),
668
669 read: Effect.fn(function* ({ id, olds, output }) {
670 const env = yield* GcpEnvironment.current;
671 const tableId = yield* toId(id, olds?.tableId, output?.tableId);
672 const datasetId = datasetIdOf(olds?.datasetId ?? output?.datasetId ?? "");
673 if (datasetId.length === 0) return undefined;
674 const existing = yield* getByRef(env.project, datasetId, tableId);
675 if (existing === undefined) return undefined;
676 const attrs = toAttrs(existing, env.project);
677 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
678 }),
679
680 list: () =>
681 Effect.gen(function* () {
682 const env = yield* GcpEnvironment.current;
683 const datasets = yield* bigquery.listDatasets
684 .pages({
685 projectId: env.project,
686 maxResults: 1000,
687 all: true,
688 })
689 .pipe(
690 Stream.flatMap((page) => Stream.fromIterable(page.datasets ?? [])),
691 Stream.runCollect,
692 Effect.map((chunk) => Array.from(chunk)),
693 Effect.catchTag("NotFound", () =>
694 Effect.succeed([] as bigquery.DatasetListDatasetsItem[]),
695 ),
696 );
697 const pages = yield* Effect.forEach(
698 datasets,
699 (dataset) => {
700 const datasetId = dataset.datasetReference?.datasetId;
701 if (datasetId === undefined) {
702 return Effect.succeed([] as ReturnType<typeof toAttrs>[]);
703 }
704 return bigquery.listTables
705 .pages({
706 projectId: env.project,
707 datasetId,
708 maxResults: 1000,
709 })
710 .pipe(
711 Stream.flatMap((page) => Stream.fromIterable(page.tables ?? [])),
712 Stream.filter((table) =>
713 Object.keys(table.labels ?? {}).some((key) => key.startsWith("alchemy-")),
714 ),
715 Stream.map((table) => toAttrs(table, env.project)),
716 Stream.runCollect,
717 Effect.map((chunk) => Array.from(chunk)),
718 Effect.catchTag("NotFound", () =>
719 Effect.succeed([] as ReturnType<typeof toAttrs>[]),
720 ),
721 );
722 },
723 { concurrency: 4 },
724 );
725 return pages.flat();
726 }),
727
728 reconcile: Effect.fn(function* ({ id, news, output }) {
729 if (news.view !== undefined && news.externalDataConfiguration !== undefined) {
730 return yield* new ConflictingTableKind({
731 message: "Set either view or externalDataConfiguration on a BigQuery table, not both",
732 });
733 }
734 const env = yield* GcpEnvironment.current;
735 const tableId = yield* toId(id, news.tableId, output?.tableId);
736 const datasetId = datasetIdOf(news.datasetId);
737 const name = resourceName(env.project, datasetId, tableId);
738 const desiredLabels = {
739 ...toLabels(news.labels),
740 ...(yield* createInternalLabels(id)),
741 };
742
743 let current = yield* getByRef(env.project, datasetId, tableId);
744
745 if (current === undefined) {
746 const created = yield* bigquery
747 .insertTables({
748 projectId: env.project,
749 datasetId,
750 body: toTableBody(env.project, datasetId, tableId, news, desiredLabels),
751 })
752 .pipe(Effect.catchTag("Conflict", () => getByRef(env.project, datasetId, tableId)));
753 current = created ?? undefined;
754 }
755
756 if (current === undefined) {
757 return yield* new TableNotResolved({ resourceName: name });
758 }
759
760 const observedLabels = tagRecord(current.labels);
761 const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
762 if (removed.length > 0) {
763 // PATCH merges labels; removing one needs a JSON null the typed body
764 // cannot carry, so replace the labels with PUT (get-modify-put).
765 current = yield* bigquery.updateTables({
766 projectId: env.project,
767 datasetId,
768 tableId,
769 body: { ...current, labels: desiredLabels },
770 });
771 }
772 const labelsChanged = removed.length === 0 && upsert.length > 0;
773 const descriptionChanged =
774 news.description !== undefined &&
775 !sameOptionalString(news.description, current.description);
776 const friendlyNameChanged =
777 news.friendlyName !== undefined &&
778 !sameOptionalString(news.friendlyName, current.friendlyName);
779 const expirationChanged =
780 news.expirationTime !== undefined &&
781 !sameOptionalString(news.expirationTime, current.expirationTime);
782 const fieldsChanged = schemaChanged(news.schema, current.schema);
783 const partitionExpirationChanged =
784 news.timePartitioning !== undefined &&
785 !sameOptionalString(
786 news.timePartitioning.expirationMs,
787 current.timePartitioning?.expirationMs,
788 );
789 const clusteringChanged =
790 news.clustering !== undefined &&
791 !jsonEqual(news.clustering.fields ?? [], current.clustering?.fields ?? []);
792 const requireFilterChanged =
793 news.requirePartitionFilter !== undefined &&
794 news.requirePartitionFilter !== (current.requirePartitionFilter === true);
795 const queryChanged = viewChanged(news.view, current.view);
796 const externalConfigChanged = externalChanged(
797 news.externalDataConfiguration,
798 current.externalDataConfiguration,
799 );
800
801 if (
802 labelsChanged ||
803 descriptionChanged ||
804 friendlyNameChanged ||
805 expirationChanged ||
806 fieldsChanged ||
807 partitionExpirationChanged ||
808 clusteringChanged ||
809 requireFilterChanged ||
810 queryChanged ||
811 externalConfigChanged
812 ) {
813 const body: bigquery.Table = {};
814 if (labelsChanged) body.labels = desiredLabels;
815 if (descriptionChanged) body.description = news.description;
816 if (friendlyNameChanged) body.friendlyName = news.friendlyName;
817 if (expirationChanged) body.expirationTime = news.expirationTime;
818 if (fieldsChanged && news.schema !== undefined) {
819 body.schema = { fields: toFieldSchema(news.schema) };
820 }
821 if (partitionExpirationChanged && news.timePartitioning !== undefined) {
822 body.timePartitioning = news.timePartitioning;
823 }
824 if (clusteringChanged && news.clustering !== undefined) {
825 body.clustering = news.clustering;
826 }
827 if (requireFilterChanged) {
828 body.requirePartitionFilter = news.requirePartitionFilter;
829 }
830 if (queryChanged && news.view !== undefined) {
831 body.view = toView(news.view);
832 }
833 if (externalConfigChanged && news.externalDataConfiguration !== undefined) {
834 body.externalDataConfiguration = toExternal(news.externalDataConfiguration);
835 }
836 current = yield* bigquery.patchTables({
837 projectId: env.project,
838 datasetId,
839 tableId,
840 body,
841 });
842 }
843
844 return toAttrs(current, env.project);
845 }),
846
847 delete: Effect.fn(function* ({ output }) {
848 yield* bigquery
849 .deleteTables({
850 projectId: output.project,
851 datasetId: output.datasetId,
852 tableId: output.tableId,
853 })
854 .pipe(Effect.catchTag("NotFound", () => Effect.void));
855 }),
856 });