The best way to control Claude Code, Codex and more from any computer or phone

main
ts 416 lines 15.7 KB
Raw
1 import * as GCP from "@alchemy.run/gcp";
2 import { expect } from "@alchemy.run/test";
3 import * as Test from "@alchemy.run/test/Alchemy";
4 import * as bigquery from "@distilled.cloud/gcp/bigquery_v2";
5 import * as Output from "alchemy/Output";
6 import * as Effect from "effect/Effect";
7 import { MinimumLogLevel } from "effect/References";
8 import * as Result from "effect/Result";
9 import * as Schedule from "effect/Schedule";
10
11 const { test } = Test.make({ providers: GCP.providers() });
12
13 const logLevel = Effect.provideService(MinimumLogLevel, process.env.DEBUG ? "Debug" : "Info");
14
15 const waitUntilGone = (projectId: string, datasetId: string, tableId: string) =>
16 bigquery
17 .getTables({
18 projectId,
19 datasetId,
20 tableId,
21 })
22 .pipe(
23 Effect.as("found" as const),
24 Effect.catchTag("NotFound", () => Effect.succeed("gone" as const)),
25 Effect.repeat({
26 schedule: Schedule.spaced("1 second"),
27 until: (status) => status === "gone",
28 times: 10,
29 }),
30 );
31
32 test.provider(
33 "create, update, and delete a table",
34 (stack) =>
35 Effect.gen(function* () {
36 yield* stack.destroy();
37
38 const created = yield* stack.deploy(
39 Effect.gen(function* () {
40 const dataset = yield* GCP.BigQuery.Dataset("Analytics", {
41 location: "US-CENTRAL1",
42 forceDestroy: true,
43 });
44 return yield* GCP.BigQuery.Table("Events", {
45 datasetId: dataset.datasetId,
46 schema: [
47 { name: "id", type: "STRING" },
48 { name: "created_at", type: "TIMESTAMP" },
49 ],
50 labels: { env: "test" },
51 description: "order events",
52 timePartitioning: { type: "DAY", field: "created_at" },
53 clustering: { fields: ["id"] },
54 });
55 }),
56 );
57
58 expect(created.tableId).toEqual(expect.any(String));
59 expect(created.tableId).toMatch(/^[a-zA-Z0-9_]+$/);
60 expect(created.datasetId).toEqual(expect.any(String));
61 expect(created.project).toEqual(expect.any(String));
62 expect(created.type).toEqual("TABLE");
63 expect(created.labels).toMatchObject({ env: "test" });
64 expect(created.description).toEqual("order events");
65 expect(created.timePartitioning).toMatchObject({
66 type: "DAY",
67 field: "created_at",
68 });
69 expect(created.clustering).toEqual({ fields: ["id"] });
70 expect(created.schema).toEqual(
71 expect.arrayContaining([
72 expect.objectContaining({ name: "id", type: "STRING" }),
73 expect.objectContaining({ name: "created_at", type: "TIMESTAMP" }),
74 ]),
75 );
76
77 const fetched = yield* bigquery.getTables({
78 projectId: created.project,
79 datasetId: created.datasetId,
80 tableId: created.tableId,
81 view: "FULL",
82 });
83 expect(fetched.tableReference?.tableId).toEqual(created.tableId);
84 expect(fetched.labels?.env).toEqual("test");
85 expect(fetched.labels?.["alchemy-id"]).toEqual(expect.any(String));
86 expect(fetched.description).toEqual("order events");
87 expect(fetched.timePartitioning?.type).toEqual("DAY");
88 expect(fetched.timePartitioning?.field).toEqual("created_at");
89 expect(fetched.clustering?.fields).toEqual(["id"]);
90
91 const inserted = yield* bigquery.insertAllTabledata({
92 projectId: created.project,
93 datasetId: created.datasetId,
94 tableId: created.tableId,
95 body: {
96 rows: [{ json: { id: "row-1", created_at: "2024-01-01T00:00:00Z" } }],
97 },
98 });
99 expect(inserted.insertErrors ?? []).toEqual([]);
100
101 const listed = yield* bigquery
102 .listTabledata({
103 projectId: created.project,
104 datasetId: created.datasetId,
105 tableId: created.tableId,
106 maxResults: 10,
107 })
108 .pipe(
109 Effect.repeat({
110 schedule: Schedule.spaced("1 second"),
111 until: (page) => (page.rows ?? []).length > 0,
112 times: 10,
113 }),
114 );
115 expect((listed.rows ?? []).length).toBeGreaterThan(0);
116
117 const updated = yield* stack.deploy(
118 Effect.gen(function* () {
119 const dataset = yield* GCP.BigQuery.Dataset("Analytics", {
120 datasetId: created.datasetId,
121 location: "US-CENTRAL1",
122 forceDestroy: true,
123 });
124 return yield* GCP.BigQuery.Table("Events", {
125 datasetId: dataset.datasetId,
126 tableId: created.tableId,
127 schema: [
128 { name: "id", type: "STRING" },
129 { name: "created_at", type: "TIMESTAMP" },
130 { name: "name", type: "STRING" },
131 ],
132 labels: { env: "prod", role: "events" },
133 description: "order events v2",
134 friendlyName: "Order Events",
135 timePartitioning: {
136 type: "DAY",
137 field: "created_at",
138 expirationMs: "2592000000",
139 },
140 clustering: { fields: ["id"] },
141 });
142 }),
143 );
144
145 expect(updated.name).toEqual(created.name);
146 expect(updated.tableId).toEqual(created.tableId);
147 expect(updated.labels).toMatchObject({ env: "prod", role: "events" });
148 expect(updated.description).toEqual("order events v2");
149 expect(updated.friendlyName).toEqual("Order Events");
150 expect(updated.schema).toEqual(
151 expect.arrayContaining([
152 expect.objectContaining({ name: "id", type: "STRING" }),
153 expect.objectContaining({ name: "created_at", type: "TIMESTAMP" }),
154 expect.objectContaining({ name: "name", type: "STRING" }),
155 ]),
156 );
157 expect(updated.timePartitioning).toMatchObject({
158 type: "DAY",
159 field: "created_at",
160 expirationMs: "2592000000",
161 });
162
163 const fetchedUpdate = yield* bigquery.getTables({
164 projectId: created.project,
165 datasetId: created.datasetId,
166 tableId: created.tableId,
167 view: "FULL",
168 });
169 expect(fetchedUpdate.labels?.env).toEqual("prod");
170 expect(fetchedUpdate.labels?.role).toEqual("events");
171 expect(fetchedUpdate.description).toEqual("order events v2");
172 expect(fetchedUpdate.friendlyName).toEqual("Order Events");
173 expect(fetchedUpdate.timePartitioning?.expirationMs).toEqual("2592000000");
174 expect((fetchedUpdate.schema?.fields ?? []).map((field) => field.name)).toEqual(
175 expect.arrayContaining(["id", "created_at", "name"]),
176 );
177
178 yield* stack.destroy();
179
180 const gone = yield* waitUntilGone(created.project, created.datasetId, created.tableId);
181 expect(gone).toEqual("gone");
182 }).pipe(logLevel),
183 { tags: ["provider:gcp", "provider:gcp:bigquery", "live"], timeout: 90_000 },
184 );
185
186 const tableStatus = (projectId: string, datasetId: string, tableId: string) =>
187 bigquery.getTables({ projectId, datasetId, tableId, view: "FULL" });
188
189 test.provider(
190 "create, update, and replace a view",
191 (stack) =>
192 Effect.gen(function* () {
193 yield* stack.destroy();
194
195 const viewStack = (props: { datasetId?: string; query?: string }) =>
196 Effect.gen(function* () {
197 const dataset = yield* GCP.BigQuery.Dataset("Analytics", {
198 datasetId: props.datasetId,
199 location: "US-CENTRAL1",
200 forceDestroy: true,
201 });
202 const events = yield* GCP.BigQuery.Table("Events", {
203 datasetId: dataset.datasetId,
204 schema: [
205 { name: "id", type: "STRING" },
206 { name: "n", type: "INT64" },
207 ],
208 });
209 return yield* GCP.BigQuery.Table("EventIds", {
210 datasetId: dataset.datasetId,
211 ...(props.query === undefined
212 ? { schema: [{ name: "id", type: "STRING" }] }
213 : {
214 view: {
215 query: Output.interpolate`${props.query} FROM ${events.datasetId}.${events.tableId}`,
216 },
217 }),
218 });
219 });
220
221 const created = yield* stack.deploy(viewStack({ query: "SELECT id" }));
222 expect(created.type).toEqual("VIEW");
223 expect(created.view?.query).toMatch(/^SELECT id FROM /);
224 expect(created.view?.useLegacySql).toEqual(false);
225
226 const fetched = yield* tableStatus(created.project, created.datasetId, created.tableId);
227 expect(fetched.type).toEqual("VIEW");
228 expect(fetched.view?.useLegacySql).toEqual(false);
229 expect((fetched.schema?.fields ?? []).map((field) => field.name)).toEqual(["id"]);
230
231 const updated = yield* stack.deploy(
232 viewStack({ datasetId: created.datasetId, query: "SELECT id, n" }),
233 );
234 expect(updated.tableId).toEqual(created.tableId);
235 expect(updated.creationTime).toEqual(created.creationTime);
236 expect(updated.view?.query).toMatch(/^SELECT id, n FROM /);
237
238 const fetchedUpdate = yield* tableStatus(created.project, created.datasetId, created.tableId);
239 expect(fetchedUpdate.creationTime).toEqual(created.creationTime);
240 expect((fetchedUpdate.schema?.fields ?? []).map((field) => field.name)).toEqual(["id", "n"]);
241
242 // A generated name is replaced create-before-delete, under a new name.
243 const replaced = yield* stack.deploy(viewStack({ datasetId: created.datasetId }));
244 expect(replaced.tableId).not.toEqual(created.tableId);
245 expect(replaced.type).toEqual("TABLE");
246 expect(replaced.view).toBeUndefined();
247
248 const fetchedReplace = yield* tableStatus(
249 replaced.project,
250 replaced.datasetId,
251 replaced.tableId,
252 );
253 expect(fetchedReplace.type).toEqual("TABLE");
254 expect(fetchedReplace.view).toBeUndefined();
255
256 const viewGone = yield* waitUntilGone(created.project, created.datasetId, created.tableId);
257 expect(viewGone).toEqual("gone");
258
259 yield* stack.destroy();
260
261 const gone = yield* waitUntilGone(replaced.project, replaced.datasetId, replaced.tableId);
262 expect(gone).toEqual("gone");
263 }).pipe(logLevel),
264 { tags: ["provider:gcp", "provider:gcp:bigquery", "live"], timeout: 120_000 },
265 );
266
267 test.provider(
268 "create and update a hive-partitioned external table over Cloud Storage",
269 (stack) =>
270 Effect.gen(function* () {
271 yield* stack.destroy();
272
273 const externalStack = (props: { datasetId?: string; requirePartitionFilter?: boolean }) =>
274 Effect.gen(function* () {
275 const bucket = yield* GCP.Storage.Bucket("Data", {
276 location: "US-CENTRAL1",
277 forceDestroy: true,
278 });
279 const first = yield* GCP.Storage.Object("DayOne", {
280 bucketName: bucket.bucketName,
281 key: "events/day=2024-01-01/part-0.json",
282 content: '{"id":"a","n":1}\n{"id":"b","n":2}\n',
283 contentType: "application/json",
284 });
285 const second = yield* GCP.Storage.Object("DayTwo", {
286 bucketName: bucket.bucketName,
287 key: "events/day=2024-01-02/part-0.json",
288 content: '{"id":"c","n":3}\n',
289 contentType: "application/json",
290 });
291 const dataset = yield* GCP.BigQuery.Dataset("Analytics", {
292 datasetId: props.datasetId,
293 location: "US-CENTRAL1",
294 forceDestroy: true,
295 });
296 // Referencing both objects orders the table after its source files,
297 // which autodetect reads at create time.
298 const table = yield* GCP.BigQuery.Table("Events", {
299 datasetId: dataset.datasetId,
300 description: "daily events",
301 externalDataConfiguration: {
302 sourceUris: [Output.interpolate`gs://${first.bucketName}/events/*`],
303 sourceFormat: "NEWLINE_DELIMITED_JSON",
304 autodetect: true,
305 hivePartitioningOptions: {
306 mode: "CUSTOM",
307 sourceUriPrefix: Output.interpolate`gs://${second.bucketName}/events/{day:DATE}`,
308 requirePartitionFilter: props.requirePartitionFilter,
309 },
310 },
311 });
312 return { bucket, table };
313 });
314
315 const { bucket, table: created } = yield* stack.deploy(externalStack({}));
316 expect(created.type).toEqual("EXTERNAL");
317 expect(created.externalDataConfiguration).toMatchObject({
318 sourceUris: [`gs://${bucket.bucketName}/events/*`],
319 sourceFormat: "NEWLINE_DELIMITED_JSON",
320 hivePartitioningOptions: {
321 mode: "CUSTOM",
322 sourceUriPrefix: `gs://${bucket.bucketName}/events/{day:DATE}`,
323 requirePartitionFilter: false,
324 },
325 });
326
327 const queried = yield* bigquery.queryJobs({
328 projectId: created.project,
329 body: {
330 query: `SELECT day, SUM(n) AS total FROM ${created.datasetId}.${created.tableId} GROUP BY day ORDER BY day`,
331 useLegacySql: false,
332 location: created.location,
333 },
334 });
335 expect(queried.jobComplete).toEqual(true);
336 expect((queried.rows ?? []).map((row) => row.f?.map((cell) => cell.v))).toEqual([
337 ["2024-01-01", "3"],
338 ["2024-01-02", "3"],
339 ]);
340
341 const { table: updated } = yield* stack.deploy(
342 externalStack({ datasetId: created.datasetId, requirePartitionFilter: true }),
343 );
344 expect(updated.tableId).toEqual(created.tableId);
345 expect(updated.creationTime).toEqual(created.creationTime);
346 expect(updated.externalDataConfiguration?.hivePartitioningOptions).toMatchObject({
347 requirePartitionFilter: true,
348 });
349
350 const fetchedUpdate = yield* tableStatus(created.project, created.datasetId, created.tableId);
351 expect(fetchedUpdate.creationTime).toEqual(created.creationTime);
352 expect(fetchedUpdate.description).toEqual("daily events");
353 expect(fetchedUpdate.externalDataConfiguration).toMatchObject({
354 sourceUris: [`gs://${bucket.bucketName}/events/*`],
355 sourceFormat: "NEWLINE_DELIMITED_JSON",
356 hivePartitioningOptions: {
357 mode: "CUSTOM",
358 sourceUriPrefix: `gs://${bucket.bucketName}/events/{day:DATE}`,
359 requirePartitionFilter: true,
360 },
361 });
362 expect((fetchedUpdate.schema?.fields ?? []).map((field) => field.name)).toEqual(
363 expect.arrayContaining(["id", "n", "day"]),
364 );
365
366 const filtered = yield* bigquery.queryJobs({
367 projectId: created.project,
368 body: {
369 query: `SELECT SUM(n) AS total FROM ${created.datasetId}.${created.tableId} WHERE day = '2024-01-01'`,
370 useLegacySql: false,
371 location: created.location,
372 },
373 });
374 expect((filtered.rows ?? []).map((row) => row.f?.map((cell) => cell.v))).toEqual([["3"]]);
375
376 yield* stack.destroy();
377
378 const gone = yield* waitUntilGone(created.project, created.datasetId, created.tableId);
379 expect(gone).toEqual("gone");
380 }).pipe(logLevel),
381 { tags: ["provider:gcp", "provider:gcp:bigquery", "live"], timeout: 180_000 },
382 );
383
384 test.provider(
385 "rejects a table that is both a view and an external table",
386 (stack) =>
387 Effect.gen(function* () {
388 yield* stack.destroy();
389
390 const result = yield* stack
391 .deploy(
392 Effect.gen(function* () {
393 const dataset = yield* GCP.BigQuery.Dataset("Analytics", {
394 location: "US-CENTRAL1",
395 forceDestroy: true,
396 });
397 return yield* GCP.BigQuery.Table("Both", {
398 datasetId: dataset.datasetId,
399 view: { query: "SELECT 1 AS id" },
400 externalDataConfiguration: {
401 sourceUris: ["gs://example-bucket/events/*"],
402 sourceFormat: "NEWLINE_DELIMITED_JSON",
403 },
404 });
405 }),
406 )
407 .pipe(Effect.result);
408 expect(Result.isFailure(result)).toBe(true);
409 if (Result.isFailure(result)) {
410 expect(result.failure).toBeInstanceOf(GCP.BigQuery.ConflictingTableKind);
411 }
412
413 yield* stack.destroy();
414 }).pipe(logLevel),
415 { tags: ["provider:gcp", "provider:gcp:bigquery", "live"], timeout: 90_000 },
416 );