@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 436 lines 16.1 KB
Raw
1 import * as rdc from "@distilled.cloud/cloudflare/r2-data-catalog";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import * as Effect from "effect/Effect";
7 import * as Predicate from "effect/Predicate";
8 import * as Redacted from "effect/Redacted";
9 import * as Schedule from "effect/Schedule";
10 import { CloudflareEnvironment } from "../CloudflareEnvironment.ts";
11 import type { Providers } from "../Providers.ts";
12 import type { Bucket } from "./Bucket.ts";
13
14 const TypeId = "Cloudflare.R2.DataCatalog" as const;
15 type TypeId = typeof TypeId;
16
17 /**
18 * Whether a maintenance job (compaction or snapshot expiration) runs.
19 */
20 export type MaintenanceState = "enabled" | "disabled";
21
22 /**
23 * Target output file size, in MB, that catalog compaction rewrites small
24 * files into.
25 */
26 export type TargetSizeMb = "64" | "128" | "256" | "512";
27
28 /**
29 * Catalog-level compaction maintenance settings.
30 */
31 export type Compaction = {
32 /**
33 * Whether compaction runs for tables in this catalog.
34 */
35 state?: MaintenanceState;
36 /**
37 * Target output file size, in MB.
38 * @default "128"
39 */
40 targetSizeMb?: TargetSizeMb;
41 };
42
43 /**
44 * Catalog-level snapshot expiration maintenance settings.
45 */
46 export type SnapshotExpiration = {
47 /**
48 * Whether snapshot expiration runs for tables in this catalog.
49 */
50 state?: MaintenanceState;
51 /**
52 * Maximum age a snapshot may reach before it is expired, expressed as a
53 * duration string (e.g. `"7d"`).
54 */
55 maxSnapshotAge?: string;
56 /**
57 * Minimum number of snapshots retained per table regardless of age.
58 */
59 minSnapshotsToKeep?: number;
60 };
61
62 export type DataCatalogProps = DataCatalogBucket & {
63 /**
64 * Compaction maintenance configuration. Only the fields you specify are
65 * enforced; omitted fields keep Cloudflare's defaults.
66 */
67 compaction?: Compaction;
68 /**
69 * Snapshot expiration maintenance configuration. Only the fields you
70 * specify are enforced; omitted fields keep Cloudflare's defaults.
71 */
72 snapshotExpiration?: SnapshotExpiration;
73 /**
74 * Cloudflare API token (with R2 read/write access) the catalog uses to
75 * run maintenance jobs against the bucket. Write-only: Cloudflare exposes
76 * only `credentialStatus: "present" | "absent"`, never the token itself.
77 * Maintenance jobs stay pending until a credential is provided.
78 */
79 token?: Redacted.Redacted<string>;
80 };
81
82 /**
83 * The bucket a catalog is enabled on. `bucketName` is the deprecated spelling
84 * of `bucket`; both name the same bucket and switching between them is not a
85 * change.
86 */
87 export type DataCatalogBucket =
88 | {
89 /**
90 * The R2 bucket to enable the Iceberg catalog on: a
91 * `Cloudflare.R2.Bucket` resource (orders the catalog after the bucket)
92 * or a bucket name. Changing the bucket replaces the catalog (the old
93 * bucket's catalog is disabled; table data is untouched).
94 */
95 bucket: string | Bucket;
96 bucketName?: never;
97 }
98 | {
99 /**
100 * Name of the R2 bucket to enable the catalog on.
101 * @deprecated Use `bucket`, which also accepts the bucket resource.
102 */
103 bucketName: string;
104 bucket?: never;
105 };
106
107 /**
108 * The bucket name behind a catalog's props: `bucket` as a name or a resolved
109 * `Cloudflare.R2.Bucket`, or the deprecated `bucketName`.
110 */
111 export const catalogBucketName = (props: unknown): string | undefined => {
112 if (props === null || typeof props !== "object") return undefined;
113 const { bucket, bucketName } = props as { bucket?: unknown; bucketName?: unknown };
114 if (typeof bucket === "string") return bucket;
115 if (bucket !== null && typeof bucket === "object") {
116 const name = (bucket as { bucketName?: unknown }).bucketName;
117 if (typeof name === "string") return name;
118 }
119 return typeof bucketName === "string" ? bucketName : undefined;
120 };
121
122 export type DataCatalogAttributes = {
123 /**
124 * Unique identifier of the catalog (stable across disable/enable cycles).
125 */
126 catalogId: string;
127 /**
128 * Catalog (warehouse) name, generated by Cloudflare as
129 * `{accountId}_{bucketName}`.
130 */
131 name: string;
132 /**
133 * Name of the R2 bucket backing the catalog.
134 */
135 bucketName: string;
136 /**
137 * The Cloudflare account the catalog belongs to.
138 */
139 accountId: string;
140 /**
141 * Catalog status. A reconciled catalog is always `active`.
142 */
143 status: "active" | "inactive";
144 /**
145 * Whether a maintenance credential is registered for this catalog.
146 */
147 credentialStatus: "present" | "absent" | (string & {});
148 /**
149 * Observed compaction maintenance configuration.
150 */
151 compaction:
152 | {
153 state: MaintenanceState;
154 targetSizeMb: TargetSizeMb;
155 }
156 | undefined;
157 /**
158 * Observed snapshot expiration maintenance configuration.
159 */
160 snapshotExpiration:
161 | {
162 state: MaintenanceState;
163 maxSnapshotAge: string;
164 minSnapshotsToKeep: number;
165 }
166 | undefined;
167 /**
168 * Iceberg REST catalog URI for this warehouse — point PyIceberg, Spark, or
169 * any Iceberg REST client at this endpoint.
170 */
171 catalogUri: string;
172 };
173
174 export type DataCatalog = Resource<
175 TypeId,
176 DataCatalogProps,
177 DataCatalogAttributes,
178 never,
179 Providers
180 >;
181
182 /**
183 * Apache Iceberg data catalog attached to a Cloudflare R2 bucket.
184 *
185 * R2 Data Catalog exposes an Iceberg REST catalog endpoint backed by an R2
186 * bucket, so engines like Spark, PyIceberg, and DuckDB can create and query
187 * Iceberg tables stored in R2. The catalog is a singleton per bucket: this
188 * resource enables it, keeps its maintenance configuration in sync, and
189 * disables it on destroy (table data in the bucket is never deleted).
190 *
191 * Also exported as `Cloudflare.Basin.Catalog` — the catalog of Cloudflare's
192 * Basin product (Pipelines + Catalog + SQL). The warehouse is named
193 * `<accountId>_<bucket>` and served at
194 * `https://catalog.cloudflarestorage.com/<accountId>/<bucket>`. Pass the
195 * catalog resource to a catalog `Cloudflare.Basin.Sink` (`catalog: Catalog`)
196 * so the sink is created after it.
197 * ### Enabling a catalog
198 * **Example:** Enable the catalog on an R2 bucket
199 * ```typescript
200 * const bucket = yield* Cloudflare.R2.Bucket("LakehouseBucket");
201 *
202 * const catalog = yield* Cloudflare.Basin.Catalog("Lakehouse", {
203 * bucket,
204 * });
205 *
206 * // Point any Iceberg REST client at the warehouse:
207 * const uri = catalog.catalogUri;
208 * const warehouse = catalog.name;
209 * ```
210 *
211 * ### Maintenance
212 * **Example:** Configure compaction and snapshot expiration
213 * ```typescript
214 * const catalog = yield* Cloudflare.Basin.Catalog("Lakehouse", {
215 * bucket,
216 * compaction: { state: "enabled", targetSizeMb: "256" },
217 * snapshotExpiration: {
218 * state: "enabled",
219 * maxSnapshotAge: "3d",
220 * minSnapshotsToKeep: 5,
221 * },
222 * });
223 * ```
224 *
225 * **Example:** Register a maintenance credential
226 * ```typescript
227 * // Maintenance jobs need an API token with R2 read/write on the bucket.
228 * const catalog = yield* Cloudflare.Basin.Catalog("Lakehouse", {
229 * bucket,
230 * compaction: { state: "enabled" },
231 * token: maintenanceToken, // Redacted<string>
232 * });
233 * ```
234 *
235 * @see https://developers.cloudflare.com/r2/data-catalog/
236 *
237 * @resource
238 * @product R2 Data Catalog
239 * @category Storage & Databases
240 */
241 export const DataCatalog = Resource<DataCatalog>(TypeId);
242
243 /**
244 * Returns true if the given value is a DataCatalog (Basin Catalog) resource.
245 */
246 export const isDataCatalog = (value: unknown): value is DataCatalog =>
247 Predicate.hasProperty(value, "Type") && value.Type === TypeId;
248
249 export const DataCatalogProvider = () =>
250 Provider.succeed(DataCatalog, {
251 stables: ["catalogId", "name", "accountId", "bucketName", "catalogUri"],
252 diff: Effect.fn(function* ({ olds, news, output }) {
253 const { accountId } = yield* yield* CloudflareEnvironment;
254 if (!isResolved(news)) return undefined;
255 if ((output?.accountId ?? accountId) !== accountId) {
256 return { action: "replace" } as const;
257 }
258 // The catalog is keyed to its bucket — moving buckets is a replacement
259 // (disable on the old bucket, enable on the new one). bucketName is an
260 // Input<string>; compare only once both sides are concrete strings.
261 const oldBucket = output?.bucketName ?? catalogBucketName(olds);
262 const newBucket = catalogBucketName(news);
263 if (oldBucket !== undefined && newBucket !== undefined && oldBucket !== newBucket) {
264 return { action: "replace" } as const;
265 }
266 return undefined;
267 }),
268 read: Effect.fn(function* ({ output, olds }) {
269 const { accountId } = yield* yield* CloudflareEnvironment;
270 const acct = output?.accountId ?? accountId;
271 const bucketName = output?.bucketName ?? catalogBucketName(olds);
272 if (bucketName === undefined) return undefined;
273
274 const observed = yield* getCatalog(acct, bucketName);
275 // `inactive` means the catalog was disabled — for reconciliation
276 // purposes the resource is gone (re-enable on the next deploy).
277 if (!observed || observed.status !== "active") return undefined;
278 const attrs = toAttributes(observed, acct);
279 // The catalog carries no ownership markers; if we have no state of our
280 // own, an active catalog may have been enabled out-of-band. Brand it
281 // `Unowned` so the engine gates takeover behind the adopt policy.
282 return output ? attrs : Unowned(attrs);
283 }),
284 list: Effect.fn(function* () {
285 const { accountId } = yield* yield* CloudflareEnvironment;
286 // R2 Data Catalog is an account-scoped collection: one warehouse per
287 // bucket that has the catalog enabled. Only `active` warehouses map to
288 // a live resource — `read` treats `inactive` (disabled) as gone.
289 return yield* rdc.listR2DataCatalogs({ accountId }).pipe(
290 Effect.map(({ warehouses }) =>
291 warehouses.filter((w) => w.status === "active").map((w) => toAttributes(w, accountId)),
292 ),
293 // Accounts without R2 Data Catalog access reject the route entirely.
294 Effect.catchTag("InvalidRoute", () => Effect.succeed([])),
295 );
296 }),
297 reconcile: Effect.fn(function* ({ news, olds, output }) {
298 const { accountId } = yield* yield* CloudflareEnvironment;
299 const acct = output?.accountId ?? accountId;
300 // Inputs have been resolved by Plan: `bucket` is a name or the
301 // bucket's attributes.
302 const bucketName = catalogBucketName(news) as string;
303
304 // Observe — `output` is only a cache; a disabled or never-enabled
305 // catalog observes as missing/inactive and falls through to ensure.
306 let observed = yield* getCatalog(acct, bucketName);
307
308 // Ensure — enable is a true idempotent upsert (re-running it on an
309 // active catalog returns the same id), so no AlreadyExists race exists.
310 // A freshly-created bucket can briefly 404 the enable endpoint
311 // (`NoSuchBucket`, code 40406) — ride out that consistency lag.
312 if (!observed || observed.status !== "active") {
313 yield* rdc.enableR2DataCatalog({ accountId: acct, bucketName }).pipe(
314 Effect.retry({
315 while: (e) => e._tag === "NoSuchBucket",
316 schedule: catalogConsistencySchedule,
317 }),
318 );
319 observed = yield* rdc.getR2DataCatalog({ accountId: acct, bucketName }).pipe(
320 // The catalog can lag behind its own enable call.
321 Effect.retry({
322 while: (e) => e._tag === "WarehouseNotFound",
323 schedule: catalogConsistencySchedule,
324 }),
325 );
326 }
327
328 // Sync credential before maintenance — Cloudflare rejects maintenance
329 // updates when the warehouse has no credential registered.
330 // The API exposes only present/absent, so `olds` serves as the rotation
331 // hint: re-push when the token value changed or no credential is
332 // registered (adoption re-pushes; idempotent).
333 let credentialStatus = (observed.credentialStatus ?? "absent") as "present" | "absent";
334 if (news.token !== undefined) {
335 const rotated =
336 olds?.token === undefined || Redacted.value(olds.token) !== Redacted.value(news.token);
337 if (credentialStatus !== "present" || rotated) {
338 yield* rdc.createCredential({
339 accountId: acct,
340 bucketName,
341 token: Redacted.value(news.token),
342 });
343 credentialStatus = "present";
344 }
345 }
346
347 // Sync maintenance — diff observed config against the fields the user
348 // actually specified; skip the API entirely on a no-op.
349 let maintenance = observed.maintenanceConfig ?? undefined;
350 const observedCompaction = maintenance?.compaction ?? undefined;
351 const observedExpiration = maintenance?.snapshotExpiration ?? undefined;
352 const compactionDirty =
353 news.compaction !== undefined &&
354 ((news.compaction.state !== undefined &&
355 news.compaction.state !== observedCompaction?.state) ||
356 (news.compaction.targetSizeMb !== undefined &&
357 news.compaction.targetSizeMb !== observedCompaction?.targetSizeMb));
358 const expirationDirty =
359 news.snapshotExpiration !== undefined &&
360 ((news.snapshotExpiration.state !== undefined &&
361 news.snapshotExpiration.state !== observedExpiration?.state) ||
362 (news.snapshotExpiration.maxSnapshotAge !== undefined &&
363 news.snapshotExpiration.maxSnapshotAge !== observedExpiration?.maxSnapshotAge) ||
364 (news.snapshotExpiration.minSnapshotsToKeep !== undefined &&
365 news.snapshotExpiration.minSnapshotsToKeep !== observedExpiration?.minSnapshotsToKeep));
366 if (compactionDirty || expirationDirty) {
367 const updated = yield* rdc.updateMaintenanceConfig({
368 accountId: acct,
369 bucketName,
370 compaction: news.compaction,
371 snapshotExpiration: news.snapshotExpiration,
372 });
373 maintenance = {
374 compaction: updated.compaction,
375 snapshotExpiration: updated.snapshotExpiration,
376 };
377 }
378
379 return toAttributes(
380 { ...observed, status: "active", maintenanceConfig: maintenance },
381 acct,
382 credentialStatus,
383 );
384 }),
385 delete: Effect.fn(function* ({ output }) {
386 // Disable is idempotent server-side; a never-enabled (or already
387 // disabled-and-bucket-deleted) catalog surfaces `WarehouseNotFound`.
388 yield* rdc
389 .disableR2DataCatalog({
390 accountId: output.accountId,
391 bucketName: output.bucketName,
392 })
393 .pipe(Effect.catchTag(["WarehouseNotFound", "NoSuchBucket"], () => Effect.void));
394 }),
395 });
396
397 // A freshly-created bucket (or freshly-enabled catalog) can briefly 404 on
398 // the r2-catalog endpoints before the warehouse propagates.
399 const catalogConsistencySchedule = Schedule.max([Schedule.exponential(100), Schedule.recurs(5)]);
400
401 /**
402 * Read a catalog by bucket name, mapping "gone" (`WarehouseNotFound`, code
403 * 40401, or a missing bucket) to `undefined`.
404 */
405 const getCatalog = (accountId: string, bucketName: string) =>
406 rdc
407 .getR2DataCatalog({ accountId, bucketName })
408 .pipe(Effect.catchTag(["WarehouseNotFound", "NoSuchBucket"], () => Effect.undefined));
409
410 const toAttributes = (
411 catalog: rdc.GetR2DataCatalogResponse,
412 accountId: string,
413 credentialStatus?: "present" | "absent",
414 ): DataCatalogAttributes => ({
415 catalogId: catalog.id,
416 name: catalog.name,
417 bucketName: catalog.bucket,
418 accountId,
419 // Distilled widens generated string enums to open unions (`string & {}`).
420 status: catalog.status as "active" | "inactive",
421 credentialStatus: credentialStatus ?? catalog.credentialStatus ?? "absent",
422 compaction: catalog.maintenanceConfig?.compaction
423 ? {
424 state: catalog.maintenanceConfig.compaction.state as MaintenanceState,
425 targetSizeMb: catalog.maintenanceConfig.compaction.targetSizeMb as TargetSizeMb,
426 }
427 : undefined,
428 snapshotExpiration: catalog.maintenanceConfig?.snapshotExpiration
429 ? {
430 state: catalog.maintenanceConfig.snapshotExpiration.state as MaintenanceState,
431 maxSnapshotAge: catalog.maintenanceConfig.snapshotExpiration.maxSnapshotAge,
432 minSnapshotsToKeep: catalog.maintenanceConfig.snapshotExpiration.minSnapshotsToKeep,
433 }
434 : undefined,
435 catalogUri: `https://catalog.cloudflarestorage.com/${accountId}/${catalog.bucket}`,
436 });