The best way to control Claude Code, Codex and more from any computer or phone

main
ts 559 lines 19.9 KB
Raw
1 import type * as Iceberg from "@distilled.cloud/iceberg";
2 import * as Data from "effect/Data";
3 import * as S from "effect/Schema";
4 import * as AST from "effect/SchemaAST";
5 import type {
6 TableColumn,
7 TableColumnType,
8 TablePrimitiveType,
9 TableSchema,
10 TableSchemaInput,
11 } from "../Table.ts";
12
13 /**
14 * An Effect Schema (or column) that has no Iceberg representation.
15 */
16 export class BasinTableSchemaUnsupported extends Data.TaggedError(
17 "Cloudflare.Basin.TableSchemaUnsupported",
18 )<{
19 message: string;
20 path: string;
21 }> {}
22
23 const unsupported = (path: string, what: string) =>
24 new BasinTableSchemaUnsupported({
25 path,
26 message: `Cloudflare.Basin.Table: column '${path || "<root>"}' is ${what}, which has no Iceberg type`,
27 });
28
29 // ---------------------------------------------------------------------------
30 // Effect Schema → column descriptor
31 // ---------------------------------------------------------------------------
32
33 const INT32_MIN = -2147483648;
34 const INT32_MAX = 2147483647;
35
36 interface CheckRep {
37 id: string;
38 payload: unknown;
39 }
40
41 const checkReps = (ast: AST.AST): CheckRep[] => {
42 const out: CheckRep[] = [];
43 const visit = (check: unknown) => {
44 const c = check as {
45 annotations?: { representation?: CheckRep };
46 checks?: readonly unknown[];
47 };
48 if (c.annotations?.representation?.id) out.push(c.annotations.representation);
49 c.checks?.forEach(visit);
50 };
51 ((ast as { checks?: readonly unknown[] }).checks ?? []).forEach(visit);
52 return out;
53 };
54
55 const hasCheck = (reps: CheckRep[], id: string) => reps.some((r) => r.id === `effect/schema/${id}`);
56
57 const numberType = (ast: AST.AST): TablePrimitiveType => {
58 const reps = checkReps(ast);
59 if (!hasCheck(reps, "isInt")) return "double";
60 const fitsInt32 = reps.some((r) => {
61 if (r.id !== "effect/schema/isBetween") return false;
62 const p = r.payload as { minimum?: number; maximum?: number } | null;
63 return (
64 typeof p?.minimum === "number" &&
65 typeof p?.maximum === "number" &&
66 p.minimum >= INT32_MIN &&
67 p.maximum <= INT32_MAX
68 );
69 });
70 return fitsInt32 ? "int" : "long";
71 };
72
73 const representationId = (ast: AST.AST): string | undefined =>
74 (ast.annotations as { representation?: { id?: string } } | undefined)?.representation?.id;
75
76 const descriptionOf = (ast: AST.AST): string | undefined => {
77 const own = (ast.annotations as { description?: unknown } | undefined)?.description;
78 if (typeof own === "string") return own;
79 if (AST.isUnion(ast)) {
80 for (const member of ast.types) {
81 const d = (member.annotations as { description?: unknown } | undefined)?.description;
82 if (typeof d === "string") return d;
83 }
84 }
85 return undefined;
86 };
87
88 const NUMERIC_RANK: Record<string, number> = { int: 0, long: 1, double: 2 };
89
90 const columnTypeOf = (ast: AST.AST, path: string): { type: TableColumnType; nullable: boolean } => {
91 switch (ast._tag) {
92 case "Suspend":
93 return columnTypeOf(ast.thunk(), path);
94 case "Union": {
95 const members = ast.types.filter(
96 (t) => !(AST.isNull(t) || AST.isUndefined(t) || AST.isVoid(t)),
97 );
98 const nullable = members.length !== ast.types.length;
99 if (members.length === 0) throw unsupported(path, "always null");
100 const resolved = members.map((m) => columnTypeOf(m, path));
101 const anyNullable = nullable || resolved.some((r) => r.nullable);
102 if (resolved.length === 1) return { type: resolved[0]!.type, nullable: anyNullable };
103 const types = resolved.map((r) => r.type);
104 if (types.every((t) => typeof t === "string" && t in NUMERIC_RANK)) {
105 const widest = types.reduce((a, b) =>
106 NUMERIC_RANK[a as string]! >= NUMERIC_RANK[b as string]! ? a : b,
107 );
108 return { type: widest, nullable: anyNullable };
109 }
110 if (types.every((t) => typeof t === "string" && t === types[0])) {
111 return { type: types[0]!, nullable: anyNullable };
112 }
113 throw unsupported(path, "a union of different types");
114 }
115 case "Literal": {
116 const literal = ast.literal;
117 if (typeof literal === "string") return { type: "string", nullable: false };
118 if (typeof literal === "boolean") return { type: "boolean", nullable: false };
119 if (typeof literal === "bigint") return { type: "long", nullable: false };
120 return { type: Number.isInteger(literal) ? "long" : "double", nullable: false };
121 }
122 case "Enum": {
123 const values = ast.enums.map(([, value]) => value);
124 if (values.every((v) => typeof v === "string")) return { type: "string", nullable: false };
125 if (values.every((v) => typeof v === "number")) return { type: "long", nullable: false };
126 throw unsupported(path, "an enum mixing strings and numbers");
127 }
128 case "TemplateLiteral":
129 return { type: "string", nullable: false };
130 case "String":
131 return {
132 type: hasCheck(checkReps(ast), "isUUID") ? "uuid" : "string",
133 nullable: false,
134 };
135 case "Number":
136 return { type: numberType(ast), nullable: false };
137 case "Boolean":
138 return { type: "boolean", nullable: false };
139 case "BigInt":
140 return { type: "long", nullable: false };
141 case "Declaration": {
142 const rep = representationId(ast);
143 if (rep === "effect/schema/Date" || rep === "effect/schema/DateTimeUtc") {
144 return { type: "timestamptz", nullable: false };
145 }
146 if (rep === "effect/schema/Uint8Array") return { type: "binary", nullable: false };
147 if (rep === "effect/schema/ReadonlyMap" && ast.typeParameters.length === 2) {
148 return {
149 type: mapTypeOf(ast.typeParameters[0]!, ast.typeParameters[1]!, path),
150 nullable: false,
151 };
152 }
153 if (rep === "effect/schema/ReadonlySet" && ast.typeParameters.length === 1) {
154 return { type: listTypeOf(ast.typeParameters[0]!, path), nullable: false };
155 }
156 // Schema.Class / TaggedClass: a declaration over the struct of its fields.
157 const [inner] = ast.typeParameters;
158 if (ast.typeParameters.length === 1 && inner !== undefined && AST.isObjects(inner)) {
159 return columnTypeOf(inner, path);
160 }
161 throw unsupported(path, `a declaration (${rep ?? "unknown"})`);
162 }
163 case "Objects": {
164 if (ast.propertySignatures.length === 0 && ast.indexSignatures.length === 1) {
165 const index = ast.indexSignatures[0]!;
166 return { type: mapTypeOf(index.parameter, index.type, path), nullable: false };
167 }
168 if (ast.indexSignatures.length > 0) {
169 throw unsupported(path, "a struct with an index signature");
170 }
171 return { type: { type: "struct", fields: structFieldsOf(ast, path) }, nullable: false };
172 }
173 case "Arrays": {
174 if (ast.elements.length === 0 && ast.rest.length === 1) {
175 return { type: listTypeOf(ast.rest[0]!, path), nullable: false };
176 }
177 throw unsupported(path, "a tuple");
178 }
179 default:
180 throw unsupported(path, `of schema type '${ast._tag}'`);
181 }
182 };
183
184 const listTypeOf = (element: AST.AST, path: string): TableColumnType => {
185 const resolved = columnTypeOf(element, `${path}.element`);
186 return { type: "list", element: resolved.type, elementRequired: !resolved.nullable };
187 };
188
189 const mapTypeOf = (key: AST.AST, value: AST.AST, path: string): TableColumnType => {
190 const k = columnTypeOf(key, `${path}.key`);
191 if (k.nullable) throw unsupported(`${path}.key`, "a nullable map key");
192 const v = columnTypeOf(value, `${path}.value`);
193 return { type: "map", key: k.type, value: v.type, valueRequired: !v.nullable };
194 };
195
196 const structFieldsOf = (ast: AST.Objects, path: string): TableColumn[] =>
197 ast.propertySignatures.map((property) => {
198 if (typeof property.name !== "string") {
199 throw unsupported(path, "keyed by a symbol");
200 }
201 const columnPath = path ? `${path}.${property.name}` : property.name;
202 const resolved = columnTypeOf(property.type, columnPath);
203 const optional = AST.isOptional(property.type);
204 const keyDoc = (property.type.context?.annotations as { description?: unknown } | undefined)
205 ?.description;
206 const doc = typeof keyDoc === "string" ? keyDoc : descriptionOf(property.type);
207 return {
208 name: property.name,
209 type: resolved.type,
210 required: !(optional || resolved.nullable),
211 ...(doc !== undefined ? { doc } : {}),
212 };
213 });
214
215 /**
216 * Convert an Effect Schema struct (or `Schema.Class`) into the plain column
217 * descriptor persisted in Table props. Plain descriptors pass through.
218 */
219 export const toTableSchema = (schema: TableSchemaInput): TableSchema => {
220 if (!S.isSchema(schema)) return schema as TableSchema;
221 const root = columnTypeOf(schema.ast, "").type;
222 if (typeof root === "string" || root.type !== "struct") {
223 throw unsupported("", "not a struct (a table schema must be a Struct or Class)");
224 }
225 return { fields: root.fields };
226 };
227
228 // ---------------------------------------------------------------------------
229 // Normalization + comparison
230 // ---------------------------------------------------------------------------
231
232 /** Canonical primitive spelling: lowercase, no whitespace. */
233 const normalizePrimitive = (t: string): string => t.replaceAll(/\s+/g, "").toLowerCase();
234
235 /** Fill defaults so two descriptors compare structurally. */
236 export const normalizeColumnType = (t: TableColumnType): TableColumnType => {
237 if (typeof t === "string") return normalizePrimitive(t) as TablePrimitiveType;
238 switch (t.type) {
239 case "struct":
240 return { type: "struct", fields: normalizeColumns(t.fields) };
241 case "list":
242 return {
243 type: "list",
244 element: normalizeColumnType(t.element),
245 elementRequired: t.elementRequired ?? false,
246 };
247 case "map":
248 return {
249 type: "map",
250 key: normalizeColumnType(t.key),
251 value: normalizeColumnType(t.value),
252 valueRequired: t.valueRequired ?? false,
253 };
254 }
255 };
256
257 export const normalizeColumns = (fields: readonly TableColumn[]): TableColumn[] =>
258 fields.map((f) => ({
259 name: f.name,
260 type: normalizeColumnType(f.type),
261 required: f.required ?? false,
262 ...(f.doc ? { doc: f.doc } : {}),
263 }));
264
265 /** Iceberg wire struct fields → column descriptors. */
266 export const fromIcebergFields = (fields: readonly Iceberg.StructField[]): TableColumn[] =>
267 fields.map((f) => ({
268 name: f.name,
269 type: fromIcebergType(f.type),
270 required: f.required,
271 ...(f.doc ? { doc: f.doc } : {}),
272 }));
273
274 const fromIcebergType = (t: Iceberg.Type): TableColumnType => {
275 if (typeof t === "string") return normalizePrimitive(t) as TablePrimitiveType;
276 switch (t.type) {
277 case "struct":
278 return { type: "struct", fields: fromIcebergFields((t as Iceberg.StructType).fields) };
279 case "list": {
280 const l = t as Iceberg.ListType;
281 return {
282 type: "list",
283 element: fromIcebergType(l.element),
284 elementRequired: l.element_required,
285 };
286 }
287 case "map": {
288 const m = t as Iceberg.MapType;
289 return {
290 type: "map",
291 key: fromIcebergType(m.key),
292 value: fromIcebergType(m.value),
293 valueRequired: m.value_required,
294 };
295 }
296 default:
297 return normalizePrimitive(t.type) as TablePrimitiveType;
298 }
299 };
300
301 const decimalOf = (t: string) => /^decimal\((\d+),(\d+)\)$/.exec(t);
302
303 /** Iceberg's legal in-place type promotions. */
304 const isPromotion = (from: string, to: string): boolean => {
305 if (from === to) return true;
306 if (from === "int" && to === "long") return true;
307 if (from === "float" && to === "double") return true;
308 const a = decimalOf(from);
309 const b = decimalOf(to);
310 return !!a && !!b && a[2] === b[2] && Number(b[1]) >= Number(a[1]);
311 };
312
313 const describe = (t: TableColumnType): string =>
314 typeof t === "string" ? t : t.type === "struct" ? "struct" : t.type;
315
316 /**
317 * Compare an observed (or previously declared) schema with the desired one
318 * and list every change Iceberg cannot apply without rewriting or losing
319 * data: dropped columns, new required columns, required-ness tightened, and
320 * type changes other than Iceberg's promotions (int→long, float→double,
321 * decimal precision widening).
322 */
323 export const destructiveChanges = (
324 observed: readonly TableColumn[],
325 desired: readonly TableColumn[],
326 ): string[] => {
327 const reasons: string[] = [];
328 compareFields("", normalizeColumns(observed), normalizeColumns(desired), reasons);
329 return reasons;
330 };
331
332 const compareFields = (
333 prefix: string,
334 observed: TableColumn[],
335 desired: TableColumn[],
336 reasons: string[],
337 ) => {
338 for (const o of observed) {
339 const path = `${prefix}${o.name}`;
340 const d = desired.find((f) => f.name === o.name);
341 if (!d) {
342 reasons.push(`drops column '${path}'`);
343 continue;
344 }
345 if (!o.required && d.required) reasons.push(`makes optional column '${path}' required`);
346 compareTypes(path, o.type, d.type, reasons);
347 }
348 for (const d of desired) {
349 if (d.required && !observed.some((f) => f.name === d.name)) {
350 reasons.push(
351 `adds required column '${prefix}${d.name}' (new columns must be optional so existing rows stay valid)`,
352 );
353 }
354 }
355 };
356
357 const compareTypes = (path: string, o: TableColumnType, d: TableColumnType, reasons: string[]) => {
358 if (typeof o === "string" || typeof d === "string") {
359 if (typeof o === "string" && typeof d === "string" && isPromotion(o, d)) return;
360 reasons.push(`changes the type of '${path}' from ${describe(o)} to ${describe(d)}`);
361 return;
362 }
363 if (o.type !== d.type) {
364 reasons.push(`changes the type of '${path}' from ${describe(o)} to ${describe(d)}`);
365 return;
366 }
367 if (o.type === "struct" && d.type === "struct") {
368 compareFields(`${path}.`, o.fields, d.fields, reasons);
369 } else if (o.type === "list" && d.type === "list") {
370 if (!o.elementRequired && d.elementRequired) {
371 reasons.push(`makes the elements of '${path}' required`);
372 }
373 compareTypes(`${path}.element`, o.element, d.element, reasons);
374 } else if (o.type === "map" && d.type === "map") {
375 if (JSON.stringify(o.key) !== JSON.stringify(d.key)) {
376 reasons.push(`changes the key type of map '${path}'`);
377 }
378 if (!o.valueRequired && d.valueRequired) {
379 reasons.push(`makes the values of '${path}' required`);
380 }
381 compareTypes(`${path}.value`, o.value, d.value, reasons);
382 }
383 };
384
385 // ---------------------------------------------------------------------------
386 // Field ids
387 // ---------------------------------------------------------------------------
388
389 /**
390 * Build the Iceberg wire schema for `desired`, reusing the field ids of
391 * columns that already exist in `observed` (matched by name at the same
392 * nesting) and assigning fresh ids from `nextId` to new columns. Ids at one
393 * struct level are assigned before descending, like Iceberg's own clients.
394 */
395 export const assignFieldIds = (
396 desired: readonly TableColumn[],
397 observed: readonly Iceberg.StructField[] | undefined,
398 nextId: () => number,
399 ): Iceberg.StructField[] => {
400 const normalized = normalizeColumns(desired);
401 const matches = normalized.map((f) => observed?.find((o) => o.name === f.name));
402 const ids = matches.map((m) => m?.id ?? nextId());
403 return normalized.map((f, i) => ({
404 id: ids[i]!,
405 name: f.name,
406 required: f.required ?? false,
407 type: buildType(f.type, matches[i]?.type, nextId),
408 ...(f.doc ? { doc: f.doc } : {}),
409 }));
410 };
411
412 const buildType = (
413 t: TableColumnType,
414 observed: Iceberg.Type | undefined,
415 nextId: () => number,
416 ): Iceberg.Type => {
417 if (typeof t === "string") return t;
418 const obs = typeof observed === "object" ? observed : undefined;
419 switch (t.type) {
420 case "struct":
421 return {
422 type: "struct",
423 fields: assignFieldIds(
424 t.fields,
425 obs?.type === "struct" ? (obs as Iceberg.StructType).fields : undefined,
426 nextId,
427 ),
428 };
429 case "list": {
430 const ol = obs?.type === "list" ? (obs as Iceberg.ListType) : undefined;
431 const elementId = ol?.element_id ?? nextId();
432 return {
433 type: "list",
434 element_id: elementId,
435 element: buildType(t.element, ol?.element, nextId),
436 element_required: t.elementRequired ?? false,
437 };
438 }
439 case "map": {
440 const om = obs?.type === "map" ? (obs as Iceberg.MapType) : undefined;
441 const keyId = om?.key_id ?? nextId();
442 const valueId = om?.value_id ?? nextId();
443 return {
444 type: "map",
445 key_id: keyId,
446 key: buildType(t.key, om?.key, nextId),
447 value_id: valueId,
448 value: buildType(t.value, om?.value, nextId),
449 value_required: t.valueRequired ?? false,
450 };
451 }
452 }
453 };
454
455 /** Highest field id anywhere in a wire schema. */
456 export const maxFieldId = (fields: readonly Iceberg.StructField[]): number => {
457 let max = 0;
458 const visitType = (t: Iceberg.Type) => {
459 if (typeof t === "string") return;
460 if (t.type === "struct") visitFields((t as Iceberg.StructType).fields);
461 else if (t.type === "list") {
462 const l = t as Iceberg.ListType;
463 max = Math.max(max, l.element_id);
464 visitType(l.element);
465 } else if (t.type === "map") {
466 const m = t as Iceberg.MapType;
467 max = Math.max(max, m.key_id, m.value_id);
468 visitType(m.key);
469 visitType(m.value);
470 }
471 };
472 const visitFields = (fs: readonly Iceberg.StructField[]) => {
473 for (const f of fs) {
474 max = Math.max(max, f.id);
475 visitType(f.type);
476 }
477 };
478 visitFields(fields);
479 return max;
480 };
481
482 /** Dotted column path → field id, for partition source resolution. */
483 export const fieldIdsByPath = (fields: readonly Iceberg.StructField[]): Map<string, number> => {
484 const out = new Map<string, number>();
485 const visit = (prefix: string, fs: readonly Iceberg.StructField[]) => {
486 for (const f of fs) {
487 const path = `${prefix}${f.name}`;
488 out.set(path, f.id);
489 if (typeof f.type === "object" && f.type.type === "struct") {
490 visit(`${path}.`, (f.type as Iceberg.StructType).fields);
491 }
492 }
493 };
494 visit("", fields);
495 return out;
496 };
497
498 // ---------------------------------------------------------------------------
499 // Partition specs
500 // ---------------------------------------------------------------------------
501
502 export interface PartitionTerm {
503 /** Dotted source column path. */
504 column: string;
505 /** Iceberg transform, e.g. `identity`, `day`, `bucket[16]`. */
506 transform: string;
507 }
508
509 const UNARY: Record<string, string> = {
510 identity: "identity",
511 year: "year",
512 years: "year",
513 month: "month",
514 months: "month",
515 day: "day",
516 days: "day",
517 date: "day",
518 hour: "hour",
519 hours: "hour",
520 void: "void",
521 };
522
523 const COLUMN = "([A-Za-z_][\\w]*(?:\\.[A-Za-z_][\\w]*)*)";
524 const UNARY_RE = new RegExp(`^(\\w+)\\(${COLUMN}\\)$`);
525 const BRACKET_RE = new RegExp(`^(bucket|truncate)\\[(\\d+)\\]\\(${COLUMN}\\)$`);
526 const ARG_RE = new RegExp(`^(bucket|truncate)\\((\\d+),${COLUMN}\\)$`);
527 const BARE_RE = new RegExp(`^${COLUMN}$`);
528
529 /**
530 * Parse a `partitionBy` term: `col`, `identity(col)`, `year|month|day|hour(col)`,
531 * `bucket[N](col)` / `bucket(N, col)`, `truncate[N](col)` / `truncate(N, col)`.
532 */
533 export const parsePartitionTerm = (term: string): PartitionTerm => {
534 const t = term.replaceAll(/\s+/g, "");
535 let m = BRACKET_RE.exec(t) ?? ARG_RE.exec(t);
536 if (m) return { column: m[3]!, transform: `${m[1]}[${m[2]}]` };
537 m = UNARY_RE.exec(t);
538 if (m && UNARY[m[1]!.toLowerCase()]) {
539 return { column: m[2]!, transform: UNARY[m[1]!.toLowerCase()]! };
540 }
541 m = BARE_RE.exec(t);
542 if (m) return { column: m[1]!, transform: "identity" };
543 throw new BasinTableSchemaUnsupported({
544 path: term,
545 message: `Cloudflare.Basin.Table: cannot parse partition term '${term}' (expected e.g. 'day(at)', 'bucket[16](id)', 'region')`,
546 });
547 };
548
549 export const partitionFieldName = ({ column, transform }: PartitionTerm): string => {
550 const base = column.replaceAll(".", "_");
551 if (transform === "identity") return base;
552 if (transform.startsWith("bucket")) return `${base}_bucket`;
553 if (transform.startsWith("truncate")) return `${base}_trunc`;
554 return `${base}_${transform}`;
555 };
556
557 export const samePartitionTerms = (a: readonly PartitionTerm[], b: readonly PartitionTerm[]) =>
558 a.length === b.length &&
559 a.every((t, i) => t.column === b[i]!.column && t.transform === b[i]!.transform);