The best way to control Claude Code, Codex and more from any computer or phone

main
ts 944 lines 32.1 KB
Raw
1 import * as basin from "@distilled.cloud/cloudflare/basin_catalog";
2 import * as Iceberg from "@distilled.cloud/iceberg";
3 import { Unowned } from "alchemy/AdoptPolicy";
4 import { deepEqual, isResolved } from "alchemy/Diff";
5 import { createPhysicalName } from "alchemy/PhysicalName";
6 import * as Provider from "alchemy/Provider";
7 import { Resource } from "alchemy/Resource";
8 import { createInternalTags, hasAlchemyTags } from "alchemy/Tags";
9 import * as Data from "effect/Data";
10 import * as Duration from "effect/Duration";
11 import * as Effect from "effect/Effect";
12 import * as Formatter from "effect/Formatter";
13 import * as Layer from "effect/Layer";
14 import * as Predicate from "effect/Predicate";
15 import type * as Redacted from "effect/Redacted";
16 import * as S from "effect/Schema";
17 import { CloudflareEnvironment } from "../CloudflareEnvironment.ts";
18 import type { Providers } from "../Providers.ts";
19 import type { DataCatalog } from "../R2/DataCatalog.ts";
20 import {
21 assignFieldIds,
22 BasinTableSchemaUnsupported,
23 destructiveChanges,
24 fieldIdsByPath,
25 fromIcebergFields,
26 maxFieldId,
27 normalizeColumns,
28 parsePartitionTerm,
29 partitionFieldName,
30 samePartitionTerms,
31 toTableSchema,
32 type PartitionTerm,
33 } from "./internal/schema.ts";
34
35 export { BasinTableSchemaUnsupported } from "./internal/schema.ts";
36
37 const TypeId = "Cloudflare.Basin.Table" as const;
38 type TypeId = typeof TypeId;
39
40 /**
41 * An Iceberg primitive type. Decimals are written `decimal(P, S)` and
42 * fixed-length binaries `fixed[N]`.
43 */
44 export type TablePrimitiveType =
45 | "boolean"
46 | "int"
47 | "long"
48 | "float"
49 | "double"
50 | "date"
51 | "time"
52 | "timestamp"
53 | "timestamptz"
54 | "timestamp_ns"
55 | "timestamptz_ns"
56 | "string"
57 | "uuid"
58 | "binary"
59 | `decimal(${number}, ${number})`
60 | `decimal(${number},${number})`
61 | `fixed[${number}]`;
62
63 /** A nested Iceberg struct column type. */
64 export interface TableStructType {
65 type: "struct";
66 /** Columns of the struct. */
67 fields: TableColumn[];
68 }
69
70 /** An Iceberg list column type. */
71 export interface TableListType {
72 type: "list";
73 /** Type of each element. */
74 element: TableColumnType;
75 /**
76 * Whether elements are non-null.
77 * @default false
78 */
79 elementRequired?: boolean;
80 }
81
82 /** An Iceberg map column type. */
83 export interface TableMapType {
84 type: "map";
85 /** Type of the keys. */
86 key: TableColumnType;
87 /** Type of the values. */
88 value: TableColumnType;
89 /**
90 * Whether values are non-null.
91 * @default false
92 */
93 valueRequired?: boolean;
94 }
95
96 /** Any Iceberg column type. */
97 export type TableColumnType = TablePrimitiveType | TableStructType | TableListType | TableMapType;
98
99 /** One column of a table schema. Field ids are assigned by Alchemy. */
100 export interface TableColumn {
101 /** Column name. */
102 name: string;
103 /** Column type. */
104 type: TableColumnType;
105 /**
106 * Whether the column is non-null.
107 * @default false
108 */
109 required?: boolean;
110 /** Column documentation stored in the Iceberg schema. */
111 doc?: string;
112 }
113
114 /**
115 * A plain Iceberg schema descriptor: the table's top-level columns.
116 */
117 export interface TableSchema {
118 /** Top-level columns. */
119 fields: TableColumn[];
120 }
121
122 /**
123 * A table schema: an Effect Schema `Struct` or `Class`, or a plain
124 * {@link TableSchema} descriptor (for types Effect Schema cannot express,
125 * like `float`, `decimal(P, S)`, or `date`).
126 */
127 export type TableSchemaInput = S.Schema<any> | TableSchema;
128
129 /**
130 * What happens to the Iceberg table when the resource is removed from the
131 * stack or the stack is destroyed.
132 *
133 * - `"retain"` — leave the table and its data in the catalog.
134 * - `"drop"` — drop the table from the catalog without purging its files.
135 * - `"purge"` — drop the table and ask the catalog to purge its data and
136 * metadata files.
137 */
138 export type TableDeletePolicy = "retain" | "drop" | "purge";
139
140 /** Target output file size, in MB, of compaction. */
141 export type TableCompactionTargetSizeMb = "64" | "128" | "256" | "512";
142
143 /** Per-table maintenance managed by the Basin catalog. */
144 export interface TableMaintenance {
145 /**
146 * Compaction rewrites small data files into larger ones. Omit to leave
147 * the table's compaction settings unmanaged.
148 */
149 compaction?: {
150 /**
151 * Whether compaction runs.
152 * @default true
153 */
154 enabled?: boolean;
155 /** Target output file size, in MB. */
156 targetSizeMb?: TableCompactionTargetSizeMb;
157 };
158 /**
159 * Snapshot expiration removes old snapshots. Omit to leave the table's
160 * snapshot expiration settings unmanaged.
161 */
162 snapshotExpiration?: {
163 /**
164 * Whether snapshot expiration runs.
165 * @default true
166 */
167 enabled?: boolean;
168 /**
169 * Expire snapshots older than this, e.g. `"7 days"` or
170 * `Duration.hours(36)`. Sent to Cloudflare in whole days, hours, or
171 * minutes.
172 */
173 olderThan?: Duration.Input;
174 /** Always keep at least this many snapshots. */
175 retainLast?: number;
176 };
177 }
178
179 export interface TableProps {
180 /**
181 * The catalog that owns the table: a `Cloudflare.Basin.Catalog` resource
182 * (orders the table after the catalog) or the name of the bucket the
183 * catalog is enabled on. The table is created in its warehouse. Changing
184 * the catalog fails the plan.
185 */
186 catalog: string | DataCatalog;
187 /**
188 * Iceberg namespace of the table. Created if it does not exist; never
189 * deleted (other tables may share it). Changing the namespace fails the
190 * plan.
191 */
192 namespace: string;
193 /**
194 * Table name. Changing it fails the plan (Iceberg renames would orphan
195 * readers and writers).
196 * @default ${app}_${id}_${stage}_${suffix} (lowercase, underscores)
197 */
198 name?: string;
199 /**
200 * Table schema. Additive changes (new optional columns, relaxing
201 * `required`, `int`→`long`, `float`→`double`, wider decimals, docs,
202 * column order) evolve the table in place. Destructive changes (dropping
203 * a column, adding a required column, narrowing or changing a type) fail
204 * the plan — Alchemy never drops data to replace a table.
205 */
206 schema: TableSchemaInput;
207 /**
208 * Partition spec as Iceberg transform terms: a bare column (`"region"`,
209 * identity), `"year(at)"`, `"month(at)"`, `"day(at)"`, `"hour(at)"`,
210 * `"bucket[16](id)"`, `"truncate[4](name)"`. Immutable: changing it fails
211 * the plan.
212 */
213 partitionBy?: string[];
214 /**
215 * Iceberg table properties. Synced in place; keys removed from this map
216 * are removed from the table. Properties set by the catalog or other
217 * writers are left alone.
218 */
219 properties?: Record<string, string>;
220 /**
221 * Per-table maintenance (compaction and snapshot expiration). Only the
222 * settings you specify are enforced. Enabling a job on a table requires
223 * the same job to be enabled on the catalog (`compaction` /
224 * `snapshotExpiration` with `state: "enabled"` on the DataCatalog).
225 */
226 maintenance?: TableMaintenance;
227 /**
228 * What happens to the table when it is removed from the stack.
229 * @default "retain"
230 */
231 delete?: TableDeletePolicy;
232 /**
233 * Bearer token for the Iceberg REST catalog. Defaults to the deploying
234 * Cloudflare credentials (API token or OAuth access token), which need
235 * R2 Data Catalog access.
236 */
237 token?: Redacted.Redacted<string>;
238 }
239
240 export interface TableAttributes {
241 /** Iceberg namespace of the table. */
242 namespace: string;
243 /** Table name. */
244 tableName: string;
245 /** `namespace.tableName`, as engines like Spark and DuckDB address it. */
246 identifier: string;
247 /** Iceberg table UUID. Stable across in-place schema evolution. */
248 tableUuid: string;
249 /** Base location of the table's files in R2. */
250 location: string | undefined;
251 /** Location of the current metadata file. */
252 metadataLocation: string | undefined;
253 /** Iceberg REST catalog URI of the warehouse. */
254 catalogUri: string;
255 /** Warehouse name (`{accountId}_{bucketName}`). */
256 warehouse: string;
257 /** The Cloudflare account that owns the catalog. */
258 accountId: string;
259 /** R2 bucket that backs the catalog. */
260 bucketName: string;
261 /** Iceberg table format version. */
262 formatVersion: number;
263 /** Id of the table's current schema. */
264 currentSchemaId: number | undefined;
265 }
266
267 export type Table = Resource<TypeId, TableProps, TableAttributes, never, Providers>;
268
269 /**
270 * A plan-time rejection: the change cannot be applied in place and Alchemy
271 * will not replace (drop) an Iceberg table to apply it.
272 */
273 export class BasinTableChangeRejected extends Data.TaggedError(
274 "Cloudflare.Basin.TableChangeRejected",
275 )<{
276 message: string;
277 table: string;
278 reasons: string[];
279 }> {}
280
281 /**
282 * The deploying Cloudflare credentials cannot authenticate against the
283 * Iceberg REST catalog (global API keys are not bearer tokens).
284 */
285 export class BasinTableCredentialsUnavailable extends Data.TaggedError(
286 "Cloudflare.Basin.TableCredentialsUnavailable",
287 )<{
288 message: string;
289 }> {}
290
291 const TableResource = Resource<Table>(TypeId);
292
293 // A schema that cannot be lowered is a defect: `Effect.sync` dies with the thrown error.
294 const toProps = <P>(props: P): Effect.Effect<P> =>
295 Effect.sync(() => {
296 const p = props as { schema?: unknown } | undefined;
297 return p?.schema !== undefined && S.isSchema(p.schema)
298 ? ({ ...p, schema: toTableSchema(p.schema as S.Schema<any>) } as P)
299 : props;
300 });
301
302 /**
303 * An Apache Iceberg table in a Cloudflare Basin (R2 Data) catalog.
304 *
305 * Declare the table's schema with Effect Schema; Alchemy derives the Iceberg
306 * schema (`String`→`string`, `Boolean`→`boolean`, `Int`→`long` (`int` when
307 * bounded to 32 bits), `Number`→`double`, `Date`→`timestamptz`,
308 * `Uint8Array`→`binary`, arrays→`list`, `Struct`/`Class`→`struct`,
309 * `Record`→`map`, optional/nullable→not required). Additive schema changes
310 * evolve the table in place, keeping its UUID and field ids; destructive
311 * changes fail the plan. By default the table is retained when removed
312 * from the stack.
313 *
314 * Writing to a table from a Pipelines sink and querying it with Basin SQL
315 * are not part of this resource yet.
316 * ### Creating a table
317 * **Example:** Table from an Effect Schema
318 * ```typescript
319 * const Order = Schema.Struct({
320 * id: Schema.String,
321 * amount: Schema.Number,
322 * at: Schema.Date,
323 * note: Schema.optional(Schema.String),
324 * });
325 *
326 * const bucket = yield* Cloudflare.R2.Bucket("Analytics");
327 * const catalog = yield* Cloudflare.R2.DataCatalog("Catalog", {
328 * bucket,
329 * });
330 * const orders = yield* Cloudflare.Basin.Table("Orders", {
331 * catalog,
332 * namespace: "sales",
333 * schema: Order,
334 * partitionBy: ["day(at)"],
335 * });
336 * // orders.identifier → "sales.<generated name>"
337 * ```
338 *
339 * **Example:** Plain Iceberg column descriptors
340 * ```typescript
341 * const prices = yield* Cloudflare.Basin.Table("Prices", {
342 * catalog,
343 * namespace: "sales",
344 * name: "prices",
345 * schema: {
346 * fields: [
347 * { name: "sku", type: "string", required: true },
348 * { name: "price", type: "decimal(10, 2)" },
349 * { name: "day", type: "date" },
350 * ],
351 * },
352 * });
353 * ```
354 *
355 * ### Evolving the schema
356 * **Example:** Add an optional column in place
357 * ```typescript
358 * const Order = Schema.Struct({
359 * id: Schema.String,
360 * amount: Schema.Number,
361 * at: Schema.Date,
362 * note: Schema.optional(Schema.String),
363 * coupon: Schema.optional(Schema.String), // new: evolves the table
364 * });
365 * ```
366 *
367 * ### Properties and maintenance
368 * **Example:** Table properties, compaction, and snapshot expiration
369 * ```typescript
370 * // Table maintenance needs the jobs enabled on the catalog first.
371 * const catalog = yield* Cloudflare.R2.DataCatalog("Catalog", {
372 * bucket,
373 * compaction: { state: "enabled" },
374 * snapshotExpiration: { state: "enabled" },
375 * token: maintenanceToken,
376 * });
377 * const orders = yield* Cloudflare.Basin.Table("Orders", {
378 * catalog,
379 * namespace: "sales",
380 * schema: Order,
381 * properties: { "write.parquet.compression-codec": "zstd" },
382 * maintenance: {
383 * compaction: { targetSizeMb: "256" },
384 * snapshotExpiration: { olderThan: "7 days", retainLast: 10 },
385 * },
386 * });
387 * ```
388 *
389 * ### Deleting
390 * **Example:** Purge the table's data on destroy
391 * ```typescript
392 * const scratch = yield* Cloudflare.Basin.Table("Scratch", {
393 * catalog,
394 * namespace: "tmp",
395 * schema: Order,
396 * delete: "purge",
397 * });
398 * ```
399 *
400 * @see https://developers.cloudflare.com/r2/data-catalog/
401 *
402 * @resource
403 * @product Basin
404 * @category Storage & Databases
405 */
406 export const Table: typeof TableResource = Object.assign(
407 (...args: [id: string, props: unknown] | [methods: object]) => {
408 if (typeof args[0] !== "string") {
409 return (TableResource as (methods: object) => unknown)(args[0]);
410 }
411 const [id, props] = args as [string, unknown];
412 const resolved = Effect.isEffect(props)
413 ? // oxlint-disable-next-line effecttsgo/unsafe-effect-type-assertion -- props are untyped; Effect-valued props are resolved by the engine
414 Effect.flatMap(props as Effect.Effect<unknown>, toProps)
415 : Effect.suspend(() => toProps(props));
416 return TableResource(id, resolved as Effect.Effect<any>);
417 },
418 TableResource,
419 ) as typeof TableResource;
420
421 /**
422 * Returns true if the given value is a Basin Table resource.
423 */
424 export const isTable = (value: unknown): value is Table =>
425 Predicate.hasProperty(value, "Type") && value.Type === TypeId;
426
427 interface CatalogRef {
428 catalogUri: string;
429 warehouse: string;
430 accountId: string;
431 bucketName: string;
432 }
433
434 const CATALOG_HOST = "https://catalog.cloudflarestorage.com";
435
436 /**
437 * The catalog prop is resolved to the DataCatalog's attributes before a
438 * lifecycle operation runs. During plan it may still be unresolved.
439 */
440 const catalogRefOf = (catalog: unknown, defaultAccountId?: string): CatalogRef | undefined => {
441 if (typeof catalog === "string") {
442 if (catalog.length === 0 || defaultAccountId === undefined) return undefined;
443 return {
444 accountId: defaultAccountId,
445 bucketName: catalog,
446 catalogUri: `${CATALOG_HOST}/${defaultAccountId}/${catalog}`,
447 warehouse: `${defaultAccountId}_${catalog}`,
448 };
449 }
450 if (catalog === null || typeof catalog !== "object") return undefined;
451 const c = catalog as Record<string, unknown>;
452 const str = (v: unknown) => (typeof v === "string" && v.length > 0 ? v : undefined);
453 const fromUri = str(c.catalogUri)?.match(/\/([^/]+)\/([^/]+)\/?$/);
454 const accountId = str(c.accountId) ?? fromUri?.[1] ?? defaultAccountId;
455 const bucketName = str(c.bucketName) ?? str(c.bucket) ?? fromUri?.[2];
456 if (!accountId || !bucketName) return undefined;
457 return {
458 accountId,
459 bucketName,
460 catalogUri: str(c.catalogUri) ?? `${CATALOG_HOST}/${accountId}/${bucketName}`,
461 warehouse: str(c.warehouse) ?? str(c.name) ?? `${accountId}_${bucketName}`,
462 };
463 };
464
465 const refFromOutput = (output: TableAttributes): CatalogRef => ({
466 catalogUri: output.catalogUri,
467 warehouse: output.warehouse,
468 accountId: output.accountId,
469 bucketName: output.bucketName,
470 });
471
472 const bearerToken = (override: Redacted.Redacted<string> | undefined) =>
473 Effect.gen(function* () {
474 if (override !== undefined) return override;
475 const creds = yield* yield* CloudflareEnvironment;
476 if (creds.type === "apiToken") return creds.apiToken;
477 if (creds.type === "oauth") return creds.accessToken;
478 return yield* new BasinTableCredentialsUnavailable({
479 message:
480 "Cloudflare.Basin.Table: the Iceberg catalog needs a bearer token; global API key credentials cannot be used. Log in with an API token or OAuth, or pass `token`.",
481 });
482 });
483
484 /** Scope Iceberg operations to one warehouse of the Basin catalog. */
485 const withCatalog =
486 (ref: CatalogRef, token: Redacted.Redacted<string>) =>
487 <A, E, R>(effect: Effect.Effect<A, E, R>) =>
488 effect.pipe(
489 Effect.provide(
490 Layer.mergeAll(
491 Iceberg.IcebergProtocol,
492 Iceberg.fromCatalogConfig({
493 uri: ref.catalogUri,
494 warehouse: ref.warehouse,
495 token,
496 }),
497 ),
498 ),
499 );
500
501 const tableNameFor = (id: string, name: string | undefined) =>
502 name !== undefined
503 ? Effect.succeed(name)
504 : createPhysicalName({ id, lowercase: true, delimiter: "_" }).pipe(
505 Effect.map((generated) => generated.replaceAll(/[^a-z0-9_]/g, "_")),
506 );
507
508 const schemaOf = (schema: TableSchemaInput) =>
509 Effect.try({
510 try: () => toTableSchema(schema),
511 catch: (error) =>
512 error instanceof BasinTableSchemaUnsupported
513 ? error
514 : new BasinTableSchemaUnsupported({ path: "", message: String(error) }),
515 });
516
517 const partitionTermsOf = (partitionBy: readonly string[] | undefined) =>
518 Effect.try({
519 try: () => (partitionBy ?? []).map(parsePartitionTerm),
520 catch: (error) =>
521 error instanceof BasinTableSchemaUnsupported
522 ? error
523 : new BasinTableSchemaUnsupported({ path: "", message: String(error) }),
524 });
525
526 const rejected = (table: string, reasons: string[]) =>
527 new BasinTableChangeRejected({
528 table,
529 reasons,
530 message: `Cloudflare.Basin.Table '${table}': this change cannot be applied in place, and Alchemy never drops an Iceberg table to replace it:\n${reasons.map((r) => ` - ${r}`).join("\n")}`,
531 });
532
533 /** HEAD the table; a missing table (or namespace) answers 404 with no body. */
534 const tableExists = (namespace: string, table: string) =>
535 Iceberg.tableExists({ namespace, table }).pipe(
536 Effect.as(true),
537 Effect.catchTag("NotFound", () => Effect.succeed(false)),
538 );
539
540 const loadTable = (namespace: string, table: string) =>
541 Iceberg.loadTable({ namespace, table }).pipe(
542 Effect.catchTag("NoSuchTableException", () => Effect.undefined),
543 );
544
545 const requirement = (type: string, fields: Record<string, unknown>): Iceberg.TableRequirement => ({
546 type,
547 ...fields,
548 });
549
550 const currentSchemaOf = (metadata: Iceberg.TableMetadata) =>
551 metadata.schemas?.find((s) => s.schema_id === metadata.current_schema_id) ??
552 metadata.schemas?.[0];
553
554 const observedPartitionTerms = (
555 metadata: Iceberg.TableMetadata,
556 fields: readonly Iceberg.StructField[],
557 ): PartitionTerm[] => {
558 const spec =
559 metadata.partition_specs?.find((s) => s.spec_id === metadata.default_spec_id) ??
560 metadata.partition_specs?.[0];
561 const pathById = new Map([...fieldIdsByPath(fields)].map(([path, fid]) => [fid, path]));
562 return (spec?.fields ?? []).map((f) => ({
563 column: pathById.get(f.source_id) ?? `#${f.source_id}`,
564 transform: f.transform.replaceAll(/\s+/g, "").toLowerCase(),
565 }));
566 };
567
568 interface Observed {
569 metadata: Iceberg.TableMetadata;
570 metadataLocation: string | undefined;
571 }
572
573 /**
574 * Converge an existing table's schema and properties onto the desired state
575 * in one optimistic commit. Observes first; skips the commit on a no-op.
576 */
577 const syncTable = (input: {
578 namespace: string;
579 table: string;
580 schema: TableSchema;
581 partition: PartitionTerm[];
582 properties: Record<string, string>;
583 ownedPropertyKeys: string[];
584 }) =>
585 Effect.gen(function* () {
586 const loaded = yield* Iceberg.loadTable({ namespace: input.namespace, table: input.table });
587 const metadata = loaded.metadata;
588 const current = currentSchemaOf(metadata);
589 const observedFields = current?.fields ?? [];
590 const identifier = `${input.namespace}.${input.table}`;
591
592 const reasons = destructiveChanges(fromIcebergFields(observedFields), input.schema.fields);
593 const observedPartition = observedPartitionTerms(metadata, observedFields);
594 if (!samePartitionTerms(observedPartition, input.partition)) {
595 reasons.push(
596 `changes the partition spec from [${observedPartition.map(describeTerm).join(", ")}] to [${input.partition.map(describeTerm).join(", ")}] (partition specs are immutable)`,
597 );
598 }
599 if (reasons.length > 0) return yield* rejected(identifier, reasons);
600
601 const requirements: Iceberg.TableRequirement[] = [
602 requirement("assert-table-uuid", { uuid: metadata.table_uuid }),
603 ];
604 const updates: Iceberg.TableUpdateInput[] = [];
605
606 if (
607 !deepEqual(
608 normalizeColumns(fromIcebergFields(observedFields)),
609 normalizeColumns(input.schema.fields),
610 )
611 ) {
612 let next = Math.max(metadata.last_column_id ?? 0, maxFieldId(observedFields)) + 1;
613 const fields = assignFieldIds(input.schema.fields, observedFields, () => next++);
614 requirements.push(
615 requirement("assert-current-schema-id", {
616 "current-schema-id": metadata.current_schema_id,
617 }),
618 );
619 updates.push(
620 { action: "add-schema", schema: { type: "struct", fields } },
621 { action: "set-current-schema", schema_id: -1 },
622 );
623 }
624
625 const observedProps = metadata.properties ?? {};
626 const changed = Object.fromEntries(
627 Object.entries(input.properties).filter(([key, value]) => observedProps[key] !== value),
628 );
629 if (Object.keys(changed).length > 0) {
630 updates.push({ action: "set-properties", updates: changed });
631 }
632 const removals = input.ownedPropertyKeys.filter(
633 (key) => !(key in input.properties) && observedProps[key] !== undefined,
634 );
635 if (removals.length > 0) {
636 updates.push({ action: "remove-properties", removals });
637 }
638
639 if (updates.length === 0) {
640 return {
641 metadata,
642 metadataLocation: loaded.metadata_location ?? undefined,
643 } satisfies Observed;
644 }
645 const committed = yield* Iceberg.updateTable({
646 namespace: input.namespace,
647 table: input.table,
648 requirements,
649 updates,
650 });
651 return {
652 metadata: committed.metadata,
653 metadataLocation: committed.metadata_location,
654 } satisfies Observed;
655 });
656
657 const describeTerm = (t: PartitionTerm) =>
658 t.transform === "identity" ? t.column : `${t.transform}(${t.column})`;
659
660 // ---------------------------------------------------------------------------
661 // Maintenance
662 // ---------------------------------------------------------------------------
663
664 const AGE_UNITS = [
665 ["d", 86_400_000],
666 ["h", 3_600_000],
667 ["m", 60_000],
668 ] as const;
669
670 /** Whole days, hours, or minutes (rounded up) — the API's `7d` / `36h` form. */
671 const formatAge = (millis: number): string => {
672 for (const [unit, size] of AGE_UNITS) {
673 if (millis % size === 0) return `${millis / size}${unit}`;
674 }
675 return `${Math.ceil(millis / 60_000)}m`;
676 };
677
678 const parseAge = (age: string | undefined): number | undefined => {
679 const m = age?.trim().match(/^(\d+)\s*([dhms])$/);
680 if (!m) return undefined;
681 const size = { d: 86_400_000, h: 3_600_000, m: 60_000, s: 1_000 }[m[2] as "d" | "h" | "m" | "s"];
682 return Number(m[1]) * size;
683 };
684
685 const syncMaintenance = (
686 ref: CatalogRef,
687 namespace: string,
688 tableName: string,
689 maintenance: TableMaintenance,
690 ) =>
691 Effect.gen(function* () {
692 const key = {
693 accountId: ref.accountId,
694 bucketName: ref.bucketName,
695 namespace,
696 tableName,
697 };
698 const { maintenanceConfig: observed } = yield* basin.getNamespacesTablesMaintenanceConfig(key);
699 const request: basin.UpdateNamespacesTablesMaintenanceConfigRequest = { ...key };
700 let dirty = false;
701
702 if (maintenance.compaction !== undefined) {
703 const desired = {
704 state: maintenance.compaction.enabled === false ? "disabled" : "enabled",
705 targetSizeMb: maintenance.compaction.targetSizeMb,
706 };
707 const o = observed.compaction ?? undefined;
708 if (
709 o?.state !== desired.state ||
710 (desired.targetSizeMb !== undefined && o?.targetSizeMb !== desired.targetSizeMb)
711 ) {
712 request.compaction = desired;
713 dirty = true;
714 }
715 }
716
717 if (maintenance.snapshotExpiration !== undefined) {
718 const se = maintenance.snapshotExpiration;
719 const olderThan =
720 se.olderThan === undefined
721 ? undefined
722 : yield* Effect.try({
723 try: () => Duration.toMillis(se.olderThan!),
724 catch: () =>
725 new BasinTableSchemaUnsupported({
726 path: "maintenance.snapshotExpiration.olderThan",
727 message: `Cloudflare.Basin.Table: invalid duration ${Formatter.format(se.olderThan)}`,
728 }),
729 });
730 const desired = {
731 state: se.enabled === false ? "disabled" : "enabled",
732 maxSnapshotAge: olderThan === undefined ? undefined : formatAge(olderThan),
733 minSnapshotsToKeep: se.retainLast,
734 };
735 const o = observed.snapshotExpiration ?? undefined;
736 if (
737 o?.state !== desired.state ||
738 (olderThan !== undefined && parseAge(o?.maxSnapshotAge) !== olderThan) ||
739 (desired.minSnapshotsToKeep !== undefined &&
740 o?.minSnapshotsToKeep !== desired.minSnapshotsToKeep)
741 ) {
742 request.snapshotExpiration = desired;
743 dirty = true;
744 }
745 }
746
747 if (dirty) yield* basin.updateNamespacesTablesMaintenanceConfig(request);
748 });
749
750 const toAttributes = (
751 ref: CatalogRef,
752 namespace: string,
753 tableName: string,
754 { metadata, metadataLocation }: Observed,
755 ): TableAttributes => ({
756 namespace,
757 tableName,
758 identifier: `${namespace}.${tableName}`,
759 tableUuid: metadata.table_uuid,
760 location: metadata.location,
761 metadataLocation,
762 catalogUri: ref.catalogUri,
763 warehouse: ref.warehouse,
764 accountId: ref.accountId,
765 bucketName: ref.bucketName,
766 formatVersion: metadata.format_version,
767 currentSchemaId: metadata.current_schema_id,
768 });
769
770 export const TableProvider = () =>
771 Provider.succeed(Table, {
772 stables: [
773 "tableUuid",
774 "namespace",
775 "tableName",
776 "identifier",
777 "location",
778 "catalogUri",
779 "warehouse",
780 "accountId",
781 "bucketName",
782 ],
783
784 diff: Effect.fn(function* ({ olds, news, output }) {
785 if (!isResolved(news)) return undefined;
786 const table = output?.identifier ?? "<new>";
787 const reasons: string[] = [];
788 if (output !== undefined) {
789 if (typeof news.namespace === "string" && news.namespace !== output.namespace) {
790 reasons.push(`changes the namespace from '${output.namespace}' to '${news.namespace}'`);
791 }
792 if (typeof news.name === "string" && news.name !== output.tableName) {
793 reasons.push(`renames the table from '${output.tableName}' to '${news.name}'`);
794 }
795 const { accountId } = yield* yield* CloudflareEnvironment;
796 const ref = isResolved(news.catalog) ? catalogRefOf(news.catalog, accountId) : undefined;
797 if (ref !== undefined && ref.warehouse !== output.warehouse) {
798 reasons.push(
799 `moves the table from warehouse '${output.warehouse}' to '${ref.warehouse}'`,
800 );
801 }
802 }
803 if (olds !== undefined) {
804 if (olds.schema !== undefined && isResolved(news.schema)) {
805 const previous = yield* schemaOf(olds.schema);
806 const next = yield* schemaOf(news.schema);
807 reasons.push(...destructiveChanges(previous.fields, next.fields));
808 }
809 if (isResolved(news.partitionBy)) {
810 const previous = yield* partitionTermsOf(olds.partitionBy);
811 const next = yield* partitionTermsOf(news.partitionBy);
812 if (!samePartitionTerms(previous, next)) {
813 reasons.push(
814 `changes the partition spec from [${previous.map(describeTerm).join(", ")}] to [${next.map(describeTerm).join(", ")}] (partition specs are immutable)`,
815 );
816 }
817 }
818 }
819 if (reasons.length > 0) return yield* rejected(table, reasons);
820 return undefined;
821 }),
822
823 read: Effect.fn(function* ({ id, olds, output }) {
824 const { accountId } = yield* yield* CloudflareEnvironment;
825 const ref = output ? refFromOutput(output) : catalogRefOf(olds?.catalog, accountId);
826 const namespace = output?.namespace ?? olds?.namespace;
827 if (ref === undefined || typeof namespace !== "string") return undefined;
828 const tableName = output?.tableName ?? (yield* tableNameFor(id, olds?.name));
829 const token = yield* bearerToken(olds?.token);
830 const loaded = yield* Effect.gen(function* () {
831 if (!(yield* tableExists(namespace, tableName))) return undefined;
832 return yield* loadTable(namespace, tableName);
833 }).pipe(withCatalog(ref, token));
834 if (loaded === undefined) return undefined;
835 const attrs = toAttributes(ref, namespace, tableName, {
836 metadata: loaded.metadata,
837 metadataLocation: loaded.metadata_location ?? undefined,
838 });
839 // Tables carry Alchemy's ownership tags as table properties.
840 if (output !== undefined) return attrs;
841 const owned = yield* hasAlchemyTags(id, loaded.metadata.properties ?? {});
842 return owned ? attrs : Unowned(attrs);
843 }),
844
845 reconcile: Effect.fn(function* ({ id, news, olds, output }) {
846 const { accountId } = yield* yield* CloudflareEnvironment;
847 const ref =
848 catalogRefOf(news.catalog, accountId) ?? (output ? refFromOutput(output) : undefined);
849 const namespace = news.namespace;
850 const tableName = yield* tableNameFor(id, news.name ?? output?.tableName);
851 const identifier = `${namespace}.${tableName}`;
852 if (ref === undefined) {
853 return yield* rejected(identifier, [
854 "has no resolved catalog (pass a Cloudflare.Basin.Catalog or a bucket name)",
855 ]);
856 }
857 if (output !== undefined && output.identifier !== identifier) {
858 return yield* rejected(output.identifier, [
859 `changes the table identifier to '${identifier}'`,
860 ]);
861 }
862
863 const token = yield* bearerToken(news.token);
864 const schema = yield* schemaOf(news.schema);
865 const partition = yield* partitionTermsOf(news.partitionBy);
866 const properties = { ...news.properties, ...(yield* createInternalTags(id)) };
867
868 const observed = yield* Effect.gen(function* () {
869 // Ensure the namespace — creation is idempotent under a race.
870 yield* Iceberg.createNamespace({ namespace: [namespace] }).pipe(
871 Effect.catchTag("AlreadyExistsException", () => Effect.void),
872 );
873
874 // Ensure the table. A concurrent create (or a create whose state
875 // write was lost) surfaces as AlreadyExists and falls through to sync.
876 if ((yield* loadTable(namespace, tableName)) === undefined) {
877 let next = 1;
878 const fields = assignFieldIds(schema.fields, undefined, () => next++);
879 const ids = fieldIdsByPath(fields);
880 const missing = partition.filter((t) => !ids.has(t.column));
881 if (missing.length > 0) {
882 return yield* rejected(
883 identifier,
884 missing.map((t) => `partitions by unknown column '${t.column}'`),
885 );
886 }
887 yield* Iceberg.createTable({
888 namespace,
889 name: tableName,
890 schema: { type: "struct", fields },
891 partition_spec:
892 partition.length > 0
893 ? {
894 fields: partition.map((t) => ({
895 source_id: ids.get(t.column)!,
896 transform: t.transform,
897 name: partitionFieldName(t),
898 })),
899 }
900 : undefined,
901 properties,
902 }).pipe(Effect.catchTag("AlreadyExistsException", () => Effect.void));
903 }
904
905 // Sync schema + properties against observed metadata; a concurrent
906 // commit invalidates the requirements — reload and retry.
907 return yield* syncTable({
908 namespace,
909 table: tableName,
910 schema,
911 partition,
912 properties,
913 ownedPropertyKeys: Object.keys(olds?.properties ?? {}),
914 }).pipe(
915 Effect.retry({
916 while: (e) => e._tag === "CommitFailedException",
917 times: 3,
918 }),
919 );
920 }).pipe(withCatalog(ref, token));
921
922 if (news.maintenance !== undefined) {
923 yield* syncMaintenance(ref, namespace, tableName, news.maintenance);
924 }
925
926 return toAttributes(ref, namespace, tableName, observed);
927 }),
928
929 delete: Effect.fn(function* ({ olds, output }) {
930 const policy = olds?.delete ?? "retain";
931 if (policy === "retain") return;
932 const token = yield* bearerToken(olds?.token);
933 // Basin answers drop on a missing table with an ambiguous 403, so
934 // observe first: HEAD is a clean 404 when the table is gone.
935 yield* Effect.gen(function* () {
936 if (!(yield* tableExists(output.namespace, output.tableName))) return;
937 yield* Iceberg.dropTable({
938 namespace: output.namespace,
939 table: output.tableName,
940 purgeRequested: policy === "purge",
941 }).pipe(Effect.catchTag("NoSuchTableException", () => Effect.void));
942 }).pipe(withCatalog(refFromOutput(output), token));
943 }),
944 });