The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 06:00 UTC
| 1 | import * as Cloudflare from "@alchemy.run/cloudflare"; |
| 2 | import * as Basin from "@alchemy.run/cloudflare/Basin"; |
| 3 | import { CloudflareEnvironment } from "@alchemy.run/cloudflare/CloudflareEnvironment"; |
| 4 | import { expect } from "@alchemy.run/test"; |
| 5 | import * as Test from "@alchemy.run/test/Alchemy"; |
| 6 | import * as basin from "@distilled.cloud/cloudflare/basin_catalog"; |
| 7 | import * as Iceberg from "@distilled.cloud/iceberg"; |
| 8 | import * as Effect from "effect/Effect"; |
| 9 | import * as Layer from "effect/Layer"; |
| 10 | import type * as Redacted from "effect/Redacted"; |
| 11 | import { MinimumLogLevel } from "effect/References"; |
| 12 | import * as Schedule from "effect/Schedule"; |
| 13 | import * as S from "effect/Schema"; |
| 14 | |
| 15 | const { test } = Test.make({ providers: Cloudflare.providers() }); |
| 16 | |
| 17 | const logLevel = Effect.provideService(MinimumLogLevel, process.env.DEBUG ? "Debug" : "Info"); |
| 18 | |
| 19 | const tags = ["provider:cloudflare", "provider:cloudflare:r2", "live"]; |
| 20 | |
| 21 | const OrderV1 = S.Struct({ |
| 22 | id: S.String, |
| 23 | amount: S.Number, |
| 24 | quantity: S.Int, |
| 25 | at: S.Date, |
| 26 | note: S.optional(S.String), |
| 27 | }); |
| 28 | |
| 29 | const OrderV2 = S.Struct({ |
| 30 | ...OrderV1.fields, |
| 31 | coupon: S.optional(S.String), |
| 32 | lines: S.optional(S.Array(S.Struct({ sku: S.String, qty: S.Int }))), |
| 33 | }); |
| 34 | |
| 35 | // Drops `amount` and `quantity`. |
| 36 | const OrderDestructive = S.Struct({ |
| 37 | id: S.String, |
| 38 | at: S.Date, |
| 39 | note: S.optional(S.String), |
| 40 | }); |
| 41 | |
| 42 | type TableOpts = Omit<Basin.TableProps, "catalog" | "namespace" | "schema"> & { |
| 43 | schema?: Basin.TableSchemaInput; |
| 44 | }; |
| 45 | |
| 46 | const apiToken = Effect.gen(function* () { |
| 47 | const creds = yield* yield* CloudflareEnvironment; |
| 48 | if (creds.type !== "apiToken") { |
| 49 | return yield* Effect.die(new Error("Basin Table tests require an API token profile")); |
| 50 | } |
| 51 | return creds.apiToken; |
| 52 | }); |
| 53 | |
| 54 | // Bucket + catalog + any number of tables, one stack program. |
| 55 | const program = ( |
| 56 | token: Redacted.Redacted<string>, |
| 57 | tables: Record<string, TableOpts>, |
| 58 | catalogMaintenance: Pick< |
| 59 | Cloudflare.R2.DataCatalogProps, |
| 60 | "compaction" | "snapshotExpiration" |
| 61 | > = {}, |
| 62 | ) => |
| 63 | Effect.gen(function* () { |
| 64 | const bucket = yield* Cloudflare.R2.Bucket("TableBucket", { forceDestroy: true }); |
| 65 | const catalog = yield* Cloudflare.R2.DataCatalog("TableCatalog", { |
| 66 | bucket, |
| 67 | token, |
| 68 | ...catalogMaintenance, |
| 69 | }); |
| 70 | const out: Record<string, Basin.Table> = {}; |
| 71 | for (const [id, opts] of Object.entries(tables)) { |
| 72 | out[id] = yield* Basin.Table(id, { |
| 73 | catalog, |
| 74 | namespace: "sales", |
| 75 | schema: OrderV1, |
| 76 | ...opts, |
| 77 | }); |
| 78 | } |
| 79 | return { bucket, catalog, tables: out }; |
| 80 | }); |
| 81 | |
| 82 | // Out-of-band Iceberg client scoped to the deployed warehouse. |
| 83 | const iceberg = |
| 84 | (catalog: { catalogUri: string; name: string }, token: Redacted.Redacted<string>) => |
| 85 | <A, E, R>(eff: Effect.Effect<A, E, R>) => |
| 86 | eff.pipe( |
| 87 | Effect.provide( |
| 88 | Layer.mergeAll( |
| 89 | Iceberg.IcebergProtocol, |
| 90 | Iceberg.fromCatalogConfig({ uri: catalog.catalogUri, warehouse: catalog.name, token }), |
| 91 | ), |
| 92 | ), |
| 93 | ); |
| 94 | |
| 95 | const currentFields = (metadata: Iceberg.TableMetadata) => |
| 96 | (metadata.schemas ?? []).find((s) => s.schema_id === metadata.current_schema_id)?.fields ?? []; |
| 97 | |
| 98 | const exists = (namespace: string, table: string) => |
| 99 | Iceberg.tableExists({ namespace, table }).pipe( |
| 100 | Effect.as(true), |
| 101 | Effect.catchTag("NotFound", () => Effect.succeed(false)), |
| 102 | ); |
| 103 | |
| 104 | test.provider( |
| 105 | "create, evolve the schema in place, sync properties, reject destructive changes", |
| 106 | (stack) => |
| 107 | Effect.gen(function* () { |
| 108 | yield* stack.destroy(); |
| 109 | const token = yield* apiToken; |
| 110 | |
| 111 | const v1 = yield* stack.deploy( |
| 112 | program(token, { |
| 113 | Orders: { |
| 114 | partitionBy: ["day(at)"], |
| 115 | properties: { "alchemy.test.a": "1", "alchemy.test.b": "2" }, |
| 116 | }, |
| 117 | }), |
| 118 | ); |
| 119 | const orders = v1.tables.Orders!; |
| 120 | expect(orders.namespace).toEqual("sales"); |
| 121 | expect(orders.identifier).toEqual(`sales.${orders.tableName}`); |
| 122 | expect(orders.tableName).toMatch(/^[a-z0-9_]+$/); |
| 123 | expect(orders.warehouse).toEqual(v1.catalog.name); |
| 124 | expect(orders.catalogUri).toEqual(v1.catalog.catalogUri); |
| 125 | |
| 126 | const withCatalog = iceberg(v1.catalog, token); |
| 127 | const live1 = yield* Iceberg.loadTable({ |
| 128 | namespace: "sales", |
| 129 | table: orders.tableName, |
| 130 | }).pipe(withCatalog); |
| 131 | expect(live1.metadata.table_uuid).toEqual(orders.tableUuid); |
| 132 | const fields1 = currentFields(live1.metadata); |
| 133 | expect(fields1.map((f) => f.name)).toEqual(["id", "amount", "quantity", "at", "note"]); |
| 134 | expect(fields1.find((f) => f.name === "at")?.type).toEqual("timestamptz"); |
| 135 | expect(fields1.find((f) => f.name === "quantity")?.type).toEqual("long"); |
| 136 | expect(fields1.find((f) => f.name === "note")?.required).toBe(false); |
| 137 | const spec = live1.metadata.partition_specs?.find( |
| 138 | (s) => s.spec_id === live1.metadata.default_spec_id, |
| 139 | ); |
| 140 | expect(spec?.fields.map((f) => f.transform)).toEqual(["day"]); |
| 141 | expect(live1.metadata.properties?.["alchemy.test.a"]).toEqual("1"); |
| 142 | expect(live1.metadata.properties?.["alchemy.test.b"]).toEqual("2"); |
| 143 | expect(live1.metadata.properties?.["alchemy::id"]).toEqual("Orders"); |
| 144 | |
| 145 | // Additive evolution + property sync: same table, new schema. |
| 146 | const v2 = yield* stack.deploy( |
| 147 | program(token, { |
| 148 | Orders: { |
| 149 | schema: OrderV2, |
| 150 | partitionBy: ["day(at)"], |
| 151 | properties: { "alchemy.test.a": "10" }, |
| 152 | }, |
| 153 | }), |
| 154 | ); |
| 155 | const orders2 = v2.tables.Orders!; |
| 156 | expect(orders2.tableUuid).toEqual(orders.tableUuid); |
| 157 | expect(orders2.currentSchemaId).not.toEqual(orders.currentSchemaId); |
| 158 | |
| 159 | const live2 = yield* Iceberg.loadTable({ |
| 160 | namespace: "sales", |
| 161 | table: orders.tableName, |
| 162 | }).pipe(withCatalog); |
| 163 | expect(live2.metadata.table_uuid).toEqual(orders.tableUuid); |
| 164 | const fields2 = currentFields(live2.metadata); |
| 165 | expect(fields2.map((f) => f.name)).toEqual([ |
| 166 | "id", |
| 167 | "amount", |
| 168 | "quantity", |
| 169 | "at", |
| 170 | "note", |
| 171 | "coupon", |
| 172 | "lines", |
| 173 | ]); |
| 174 | // Existing columns keep their field ids. |
| 175 | for (const f of fields1) { |
| 176 | expect(fields2.find((g) => g.name === f.name)?.id).toEqual(f.id); |
| 177 | } |
| 178 | expect(fields2.find((f) => f.name === "coupon")?.required).toBe(false); |
| 179 | expect(live2.metadata.properties?.["alchemy.test.a"]).toEqual("10"); |
| 180 | expect(live2.metadata.properties?.["alchemy.test.b"]).toBeUndefined(); |
| 181 | |
| 182 | // Re-deploying the same program is a no-op commit. |
| 183 | const v2b = yield* stack.deploy( |
| 184 | program(token, { |
| 185 | Orders: { |
| 186 | schema: OrderV2, |
| 187 | partitionBy: ["day(at)"], |
| 188 | properties: { "alchemy.test.a": "10" }, |
| 189 | }, |
| 190 | }), |
| 191 | ); |
| 192 | expect(v2b.tables.Orders!.metadataLocation).toEqual(orders2.metadataLocation); |
| 193 | |
| 194 | // Destructive schema change fails the plan. |
| 195 | const destructive = yield* stack |
| 196 | .deploy( |
| 197 | program(token, { |
| 198 | Orders: { schema: OrderDestructive, partitionBy: ["day(at)"] }, |
| 199 | }), |
| 200 | ) |
| 201 | .pipe(Effect.flip); |
| 202 | expect(destructive).toBeInstanceOf(Basin.BasinTableChangeRejected); |
| 203 | expect((destructive as Basin.BasinTableChangeRejected).reasons).toContain( |
| 204 | "drops column 'amount'", |
| 205 | ); |
| 206 | |
| 207 | // Partition specs are immutable. |
| 208 | const repartition = yield* stack |
| 209 | .deploy( |
| 210 | program(token, { |
| 211 | Orders: { schema: OrderV2, partitionBy: ["month(at)"] }, |
| 212 | }), |
| 213 | ) |
| 214 | .pipe(Effect.flip); |
| 215 | expect(repartition).toBeInstanceOf(Basin.BasinTableChangeRejected); |
| 216 | |
| 217 | // The table is untouched by the rejected plans. |
| 218 | const live3 = yield* Iceberg.loadTable({ |
| 219 | namespace: "sales", |
| 220 | table: orders.tableName, |
| 221 | }).pipe(withCatalog); |
| 222 | expect(live3.metadata.metadata_log?.length).toEqual(live2.metadata.metadata_log?.length); |
| 223 | expect(currentFields(live3.metadata).map((f) => f.name)).toContain("amount"); |
| 224 | |
| 225 | yield* stack.destroy(); |
| 226 | }).pipe(logLevel), |
| 227 | { tags, timeout: 120_000 }, |
| 228 | ); |
| 229 | |
| 230 | // Table-level maintenance requires the job to be enabled on the catalog. |
| 231 | const catalogMaintenance = { |
| 232 | compaction: { state: "enabled" }, |
| 233 | snapshotExpiration: { state: "enabled" }, |
| 234 | } as const; |
| 235 | |
| 236 | test.provider( |
| 237 | "syncs per-table maintenance", |
| 238 | (stack) => |
| 239 | Effect.gen(function* () { |
| 240 | yield* stack.destroy(); |
| 241 | const token = yield* apiToken; |
| 242 | |
| 243 | const v1 = yield* stack.deploy( |
| 244 | program( |
| 245 | token, |
| 246 | { |
| 247 | Maintained: { |
| 248 | maintenance: { |
| 249 | compaction: { targetSizeMb: "256" }, |
| 250 | snapshotExpiration: { olderThan: "3 days", retainLast: 5 }, |
| 251 | }, |
| 252 | }, |
| 253 | }, |
| 254 | catalogMaintenance, |
| 255 | ), |
| 256 | ); |
| 257 | const table = v1.tables.Maintained!; |
| 258 | const key = { |
| 259 | accountId: table.accountId, |
| 260 | bucketName: table.bucketName, |
| 261 | namespace: table.namespace, |
| 262 | tableName: table.tableName, |
| 263 | }; |
| 264 | const live1 = yield* basin.getNamespacesTablesMaintenanceConfig(key); |
| 265 | expect(live1.maintenanceConfig.compaction?.state).toEqual("enabled"); |
| 266 | expect(live1.maintenanceConfig.compaction?.targetSizeMb).toEqual("256"); |
| 267 | expect(live1.maintenanceConfig.snapshotExpiration?.state).toEqual("enabled"); |
| 268 | expect(live1.maintenanceConfig.snapshotExpiration?.maxSnapshotAge).toEqual("3d"); |
| 269 | expect(live1.maintenanceConfig.snapshotExpiration?.minSnapshotsToKeep).toEqual(5); |
| 270 | |
| 271 | yield* stack.deploy( |
| 272 | program( |
| 273 | token, |
| 274 | { |
| 275 | Maintained: { |
| 276 | maintenance: { |
| 277 | compaction: { enabled: false }, |
| 278 | snapshotExpiration: { olderThan: "36 hours", retainLast: 7 }, |
| 279 | }, |
| 280 | }, |
| 281 | }, |
| 282 | catalogMaintenance, |
| 283 | ), |
| 284 | ); |
| 285 | const live2 = yield* basin.getNamespacesTablesMaintenanceConfig(key); |
| 286 | expect(live2.maintenanceConfig.compaction?.state).toEqual("disabled"); |
| 287 | expect(live2.maintenanceConfig.snapshotExpiration?.maxSnapshotAge).toEqual("36h"); |
| 288 | expect(live2.maintenanceConfig.snapshotExpiration?.minSnapshotsToKeep).toEqual(7); |
| 289 | |
| 290 | yield* stack.destroy(); |
| 291 | }).pipe(logLevel), |
| 292 | { tags, timeout: 120_000 }, |
| 293 | ); |
| 294 | |
| 295 | test.provider( |
| 296 | "delete policies: retain leaves the table, drop and purge remove it", |
| 297 | (stack) => |
| 298 | Effect.gen(function* () { |
| 299 | yield* stack.destroy(); |
| 300 | const token = yield* apiToken; |
| 301 | |
| 302 | const deployed = yield* stack.deploy( |
| 303 | program(token, { |
| 304 | Retained: {}, |
| 305 | Dropped: { delete: "drop" }, |
| 306 | Purged: { delete: "purge" }, |
| 307 | }), |
| 308 | ); |
| 309 | const withCatalog = iceberg(deployed.catalog, token); |
| 310 | const { Retained, Dropped, Purged } = deployed.tables; |
| 311 | for (const t of [Retained!, Dropped!, Purged!]) { |
| 312 | expect(yield* exists("sales", t.tableName).pipe(withCatalog)).toBe(true); |
| 313 | } |
| 314 | |
| 315 | // Remove the tables from the stack; keep the bucket and catalog. |
| 316 | yield* stack.deploy(program(token, {})); |
| 317 | |
| 318 | // Catalog deletes are eventually consistent — poll briefly. |
| 319 | const settled = (table: string, expected: boolean) => |
| 320 | exists("sales", table).pipe( |
| 321 | withCatalog, |
| 322 | Effect.repeat({ |
| 323 | schedule: Schedule.spaced("1 second"), |
| 324 | until: (present) => present === expected, |
| 325 | times: 10, |
| 326 | }), |
| 327 | ); |
| 328 | expect(yield* settled(Retained!.tableName, true)).toBe(true); |
| 329 | expect(yield* settled(Dropped!.tableName, false)).toBe(false); |
| 330 | expect(yield* settled(Purged!.tableName, false)).toBe(false); |
| 331 | |
| 332 | // Destroy disables the catalog and empties + deletes the bucket, |
| 333 | // taking the retained table with it. |
| 334 | yield* stack.destroy(); |
| 335 | }).pipe(logLevel), |
| 336 | { tags, timeout: 120_000 }, |
| 337 | ); |