The best way to control Claude Code, Codex and more from any computer or phone

main
ts 337 lines 11.6 KB
Raw
1 import * as Cloudflare from "@alchemy.run/cloudflare";
2 import * as Basin from "@alchemy.run/cloudflare/Basin";
3 import { CloudflareEnvironment } from "@alchemy.run/cloudflare/CloudflareEnvironment";
4 import { expect } from "@alchemy.run/test";
5 import * as Test from "@alchemy.run/test/Alchemy";
6 import * as basin from "@distilled.cloud/cloudflare/basin_catalog";
7 import * as Iceberg from "@distilled.cloud/iceberg";
8 import * as Effect from "effect/Effect";
9 import * as Layer from "effect/Layer";
10 import type * as Redacted from "effect/Redacted";
11 import { MinimumLogLevel } from "effect/References";
12 import * as Schedule from "effect/Schedule";
13 import * as S from "effect/Schema";
14
15 const { test } = Test.make({ providers: Cloudflare.providers() });
16
17 const logLevel = Effect.provideService(MinimumLogLevel, process.env.DEBUG ? "Debug" : "Info");
18
19 const tags = ["provider:cloudflare", "provider:cloudflare:r2", "live"];
20
21 const OrderV1 = S.Struct({
22 id: S.String,
23 amount: S.Number,
24 quantity: S.Int,
25 at: S.Date,
26 note: S.optional(S.String),
27 });
28
29 const OrderV2 = S.Struct({
30 ...OrderV1.fields,
31 coupon: S.optional(S.String),
32 lines: S.optional(S.Array(S.Struct({ sku: S.String, qty: S.Int }))),
33 });
34
35 // Drops `amount` and `quantity`.
36 const OrderDestructive = S.Struct({
37 id: S.String,
38 at: S.Date,
39 note: S.optional(S.String),
40 });
41
42 type TableOpts = Omit<Basin.TableProps, "catalog" | "namespace" | "schema"> & {
43 schema?: Basin.TableSchemaInput;
44 };
45
46 const apiToken = Effect.gen(function* () {
47 const creds = yield* yield* CloudflareEnvironment;
48 if (creds.type !== "apiToken") {
49 return yield* Effect.die(new Error("Basin Table tests require an API token profile"));
50 }
51 return creds.apiToken;
52 });
53
54 // Bucket + catalog + any number of tables, one stack program.
55 const program = (
56 token: Redacted.Redacted<string>,
57 tables: Record<string, TableOpts>,
58 catalogMaintenance: Pick<
59 Cloudflare.R2.DataCatalogProps,
60 "compaction" | "snapshotExpiration"
61 > = {},
62 ) =>
63 Effect.gen(function* () {
64 const bucket = yield* Cloudflare.R2.Bucket("TableBucket", { forceDestroy: true });
65 const catalog = yield* Cloudflare.R2.DataCatalog("TableCatalog", {
66 bucket,
67 token,
68 ...catalogMaintenance,
69 });
70 const out: Record<string, Basin.Table> = {};
71 for (const [id, opts] of Object.entries(tables)) {
72 out[id] = yield* Basin.Table(id, {
73 catalog,
74 namespace: "sales",
75 schema: OrderV1,
76 ...opts,
77 });
78 }
79 return { bucket, catalog, tables: out };
80 });
81
82 // Out-of-band Iceberg client scoped to the deployed warehouse.
83 const iceberg =
84 (catalog: { catalogUri: string; name: string }, token: Redacted.Redacted<string>) =>
85 <A, E, R>(eff: Effect.Effect<A, E, R>) =>
86 eff.pipe(
87 Effect.provide(
88 Layer.mergeAll(
89 Iceberg.IcebergProtocol,
90 Iceberg.fromCatalogConfig({ uri: catalog.catalogUri, warehouse: catalog.name, token }),
91 ),
92 ),
93 );
94
95 const currentFields = (metadata: Iceberg.TableMetadata) =>
96 (metadata.schemas ?? []).find((s) => s.schema_id === metadata.current_schema_id)?.fields ?? [];
97
98 const exists = (namespace: string, table: string) =>
99 Iceberg.tableExists({ namespace, table }).pipe(
100 Effect.as(true),
101 Effect.catchTag("NotFound", () => Effect.succeed(false)),
102 );
103
104 test.provider(
105 "create, evolve the schema in place, sync properties, reject destructive changes",
106 (stack) =>
107 Effect.gen(function* () {
108 yield* stack.destroy();
109 const token = yield* apiToken;
110
111 const v1 = yield* stack.deploy(
112 program(token, {
113 Orders: {
114 partitionBy: ["day(at)"],
115 properties: { "alchemy.test.a": "1", "alchemy.test.b": "2" },
116 },
117 }),
118 );
119 const orders = v1.tables.Orders!;
120 expect(orders.namespace).toEqual("sales");
121 expect(orders.identifier).toEqual(`sales.${orders.tableName}`);
122 expect(orders.tableName).toMatch(/^[a-z0-9_]+$/);
123 expect(orders.warehouse).toEqual(v1.catalog.name);
124 expect(orders.catalogUri).toEqual(v1.catalog.catalogUri);
125
126 const withCatalog = iceberg(v1.catalog, token);
127 const live1 = yield* Iceberg.loadTable({
128 namespace: "sales",
129 table: orders.tableName,
130 }).pipe(withCatalog);
131 expect(live1.metadata.table_uuid).toEqual(orders.tableUuid);
132 const fields1 = currentFields(live1.metadata);
133 expect(fields1.map((f) => f.name)).toEqual(["id", "amount", "quantity", "at", "note"]);
134 expect(fields1.find((f) => f.name === "at")?.type).toEqual("timestamptz");
135 expect(fields1.find((f) => f.name === "quantity")?.type).toEqual("long");
136 expect(fields1.find((f) => f.name === "note")?.required).toBe(false);
137 const spec = live1.metadata.partition_specs?.find(
138 (s) => s.spec_id === live1.metadata.default_spec_id,
139 );
140 expect(spec?.fields.map((f) => f.transform)).toEqual(["day"]);
141 expect(live1.metadata.properties?.["alchemy.test.a"]).toEqual("1");
142 expect(live1.metadata.properties?.["alchemy.test.b"]).toEqual("2");
143 expect(live1.metadata.properties?.["alchemy::id"]).toEqual("Orders");
144
145 // Additive evolution + property sync: same table, new schema.
146 const v2 = yield* stack.deploy(
147 program(token, {
148 Orders: {
149 schema: OrderV2,
150 partitionBy: ["day(at)"],
151 properties: { "alchemy.test.a": "10" },
152 },
153 }),
154 );
155 const orders2 = v2.tables.Orders!;
156 expect(orders2.tableUuid).toEqual(orders.tableUuid);
157 expect(orders2.currentSchemaId).not.toEqual(orders.currentSchemaId);
158
159 const live2 = yield* Iceberg.loadTable({
160 namespace: "sales",
161 table: orders.tableName,
162 }).pipe(withCatalog);
163 expect(live2.metadata.table_uuid).toEqual(orders.tableUuid);
164 const fields2 = currentFields(live2.metadata);
165 expect(fields2.map((f) => f.name)).toEqual([
166 "id",
167 "amount",
168 "quantity",
169 "at",
170 "note",
171 "coupon",
172 "lines",
173 ]);
174 // Existing columns keep their field ids.
175 for (const f of fields1) {
176 expect(fields2.find((g) => g.name === f.name)?.id).toEqual(f.id);
177 }
178 expect(fields2.find((f) => f.name === "coupon")?.required).toBe(false);
179 expect(live2.metadata.properties?.["alchemy.test.a"]).toEqual("10");
180 expect(live2.metadata.properties?.["alchemy.test.b"]).toBeUndefined();
181
182 // Re-deploying the same program is a no-op commit.
183 const v2b = yield* stack.deploy(
184 program(token, {
185 Orders: {
186 schema: OrderV2,
187 partitionBy: ["day(at)"],
188 properties: { "alchemy.test.a": "10" },
189 },
190 }),
191 );
192 expect(v2b.tables.Orders!.metadataLocation).toEqual(orders2.metadataLocation);
193
194 // Destructive schema change fails the plan.
195 const destructive = yield* stack
196 .deploy(
197 program(token, {
198 Orders: { schema: OrderDestructive, partitionBy: ["day(at)"] },
199 }),
200 )
201 .pipe(Effect.flip);
202 expect(destructive).toBeInstanceOf(Basin.BasinTableChangeRejected);
203 expect((destructive as Basin.BasinTableChangeRejected).reasons).toContain(
204 "drops column 'amount'",
205 );
206
207 // Partition specs are immutable.
208 const repartition = yield* stack
209 .deploy(
210 program(token, {
211 Orders: { schema: OrderV2, partitionBy: ["month(at)"] },
212 }),
213 )
214 .pipe(Effect.flip);
215 expect(repartition).toBeInstanceOf(Basin.BasinTableChangeRejected);
216
217 // The table is untouched by the rejected plans.
218 const live3 = yield* Iceberg.loadTable({
219 namespace: "sales",
220 table: orders.tableName,
221 }).pipe(withCatalog);
222 expect(live3.metadata.metadata_log?.length).toEqual(live2.metadata.metadata_log?.length);
223 expect(currentFields(live3.metadata).map((f) => f.name)).toContain("amount");
224
225 yield* stack.destroy();
226 }).pipe(logLevel),
227 { tags, timeout: 120_000 },
228 );
229
230 // Table-level maintenance requires the job to be enabled on the catalog.
231 const catalogMaintenance = {
232 compaction: { state: "enabled" },
233 snapshotExpiration: { state: "enabled" },
234 } as const;
235
236 test.provider(
237 "syncs per-table maintenance",
238 (stack) =>
239 Effect.gen(function* () {
240 yield* stack.destroy();
241 const token = yield* apiToken;
242
243 const v1 = yield* stack.deploy(
244 program(
245 token,
246 {
247 Maintained: {
248 maintenance: {
249 compaction: { targetSizeMb: "256" },
250 snapshotExpiration: { olderThan: "3 days", retainLast: 5 },
251 },
252 },
253 },
254 catalogMaintenance,
255 ),
256 );
257 const table = v1.tables.Maintained!;
258 const key = {
259 accountId: table.accountId,
260 bucketName: table.bucketName,
261 namespace: table.namespace,
262 tableName: table.tableName,
263 };
264 const live1 = yield* basin.getNamespacesTablesMaintenanceConfig(key);
265 expect(live1.maintenanceConfig.compaction?.state).toEqual("enabled");
266 expect(live1.maintenanceConfig.compaction?.targetSizeMb).toEqual("256");
267 expect(live1.maintenanceConfig.snapshotExpiration?.state).toEqual("enabled");
268 expect(live1.maintenanceConfig.snapshotExpiration?.maxSnapshotAge).toEqual("3d");
269 expect(live1.maintenanceConfig.snapshotExpiration?.minSnapshotsToKeep).toEqual(5);
270
271 yield* stack.deploy(
272 program(
273 token,
274 {
275 Maintained: {
276 maintenance: {
277 compaction: { enabled: false },
278 snapshotExpiration: { olderThan: "36 hours", retainLast: 7 },
279 },
280 },
281 },
282 catalogMaintenance,
283 ),
284 );
285 const live2 = yield* basin.getNamespacesTablesMaintenanceConfig(key);
286 expect(live2.maintenanceConfig.compaction?.state).toEqual("disabled");
287 expect(live2.maintenanceConfig.snapshotExpiration?.maxSnapshotAge).toEqual("36h");
288 expect(live2.maintenanceConfig.snapshotExpiration?.minSnapshotsToKeep).toEqual(7);
289
290 yield* stack.destroy();
291 }).pipe(logLevel),
292 { tags, timeout: 120_000 },
293 );
294
295 test.provider(
296 "delete policies: retain leaves the table, drop and purge remove it",
297 (stack) =>
298 Effect.gen(function* () {
299 yield* stack.destroy();
300 const token = yield* apiToken;
301
302 const deployed = yield* stack.deploy(
303 program(token, {
304 Retained: {},
305 Dropped: { delete: "drop" },
306 Purged: { delete: "purge" },
307 }),
308 );
309 const withCatalog = iceberg(deployed.catalog, token);
310 const { Retained, Dropped, Purged } = deployed.tables;
311 for (const t of [Retained!, Dropped!, Purged!]) {
312 expect(yield* exists("sales", t.tableName).pipe(withCatalog)).toBe(true);
313 }
314
315 // Remove the tables from the stack; keep the bucket and catalog.
316 yield* stack.deploy(program(token, {}));
317
318 // Catalog deletes are eventually consistent — poll briefly.
319 const settled = (table: string, expected: boolean) =>
320 exists("sales", table).pipe(
321 withCatalog,
322 Effect.repeat({
323 schedule: Schedule.spaced("1 second"),
324 until: (present) => present === expected,
325 times: 10,
326 }),
327 );
328 expect(yield* settled(Retained!.tableName, true)).toBe(true);
329 expect(yield* settled(Dropped!.tableName, false)).toBe(false);
330 expect(yield* settled(Purged!.tableName, false)).toBe(false);
331
332 // Destroy disables the catalog and empties + deletes the bucket,
333 // taking the retained table with it.
334 yield* stack.destroy();
335 }).pipe(logLevel),
336 { tags, timeout: 120_000 },
337 );