The best way to control Claude Code, Codex and more from any computer or phone

main
ts 503 lines 16.7 KB
Raw
1 import * as bigquery from "@distilled.cloud/gcp/bigquery_v2";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import { createPhysicalName } from "alchemy/PhysicalName";
5 import * as Provider from "alchemy/Provider";
6 import { Resource } from "alchemy/Resource";
7 import { tagRecord } from "alchemy/Tags";
8 import * as Data from "effect/Data";
9 import * as Effect from "effect/Effect";
10 import * as Schedule from "effect/Schedule";
11 import * as Stream from "effect/Stream";
12 import { GcpEnvironment } from "../Environment.ts";
13 import {
14 createInternalLabels,
15 hasAlchemyLabels,
16 stripInternalLabels,
17 toLabels,
18 } from "../Labels.ts";
19 import type { Providers } from "../Providers.ts";
20
21 const MAX_JOB_ID_LENGTH = 1024;
22
23 export type JobQuery = bigquery.JobConfigurationQuery;
24 export type JobLoad = bigquery.JobConfigurationLoad;
25 export type JobCopy = bigquery.JobConfigurationTableCopy;
26 export type JobExtract = bigquery.JobConfigurationExtract;
27 export type JobError = bigquery.ErrorProto;
28
29 export type JobProps = {
30 /**
31 * Job id (the `{jobId}` segment of `projects/{project}/jobs/{jobId}`).
32 * If omitted, a unique name is generated from the stack, stage, and
33 * logical id. Letters, numbers, underscores, and dashes; max 1,024
34 * characters. Immutable — changing it replaces the job.
35 */
36 jobId?: string;
37 /**
38 * Geographic location (`US`, `EU`, `US-CENTRAL1`, `us-central1`, …).
39 * Immutable — changing it replaces the job.
40 * @default the stack's GCP region (`GCP.Region`, profile region, `us-central1`)
41 */
42 location?: string;
43 /**
44 * User labels. Alchemy ownership labels are merged in automatically.
45 * Immutable after create — changing labels replaces the job.
46 */
47 labels?: Record<string, string>;
48 /**
49 * Query job configuration. Mutually exclusive with `load`, `copy`, and
50 * `extract`. GoogleSQL is the default (`useLegacySql: false`).
51 */
52 query?: JobQuery;
53 /**
54 * Load job configuration. Mutually exclusive with `query`, `copy`, and
55 * `extract`.
56 */
57 load?: JobLoad;
58 /**
59 * Copy job configuration. Mutually exclusive with `query`, `load`, and
60 * `extract`.
61 */
62 copy?: JobCopy;
63 /**
64 * Extract job configuration. Mutually exclusive with `query`, `load`,
65 * and `copy`.
66 */
67 extract?: JobExtract;
68 /**
69 * Job timeout in milliseconds relative to creation time.
70 */
71 jobTimeoutMs?: string;
72 /**
73 * Reservation the job should run on, as
74 * `projects/{project}/locations/{location}/reservations/{reservation}`.
75 */
76 reservation?: string;
77 };
78
79 export type Job = Resource<
80 "GCP.BigQuery.Job",
81 JobProps,
82 {
83 /** Resource path `projects/{project}/jobs/{jobId}`. */
84 name: string;
85 /** Opaque id `project:jobId`. */
86 id: string;
87 /** Job id. */
88 jobId: string;
89 /** Project id. */
90 project: string;
91 /** Geographic location. */
92 location: string;
93 /** User labels (Alchemy ownership labels stripped). */
94 labels: Record<string, string>;
95 /** Job kind (`QUERY`, `LOAD`, `EXTRACT`, `COPY`, …). */
96 jobType: string | undefined;
97 /** Running state (`PENDING`, `RUNNING`, `DONE`). */
98 state: string | undefined;
99 /** Final error when the job completed unsuccessfully. */
100 errorResult: JobError | undefined;
101 /** Query text when this is a query job. */
102 query: string | undefined;
103 /** Self-link URL. */
104 selfLink: string | undefined;
105 /** Creator email. */
106 userEmail: string | undefined;
107 /** Creation time in milliseconds since epoch. */
108 creationTime: string | undefined;
109 /** Start time in milliseconds since epoch. */
110 startTime: string | undefined;
111 /** End time in milliseconds since epoch. */
112 endTime: string | undefined;
113 },
114 never,
115 Providers
116 >;
117
118 /**
119 * A Google BigQuery job — a one-shot query, load, copy, or extract.
120 *
121 * Jobs are immutable after insert. Changing `jobId`, `location`, labels,
122 * or the job configuration replaces the job. BigQuery job ids cannot be
123 * reused even after metadata delete, so replacement always creates a new
124 * id then deletes the previous generation. Destroy cancels a running job,
125 * then deletes its metadata.
126 *
127 * ### Creating a Job
128 * **Example:** GoogleSQL query
129 * ```typescript
130 * const job = yield* GCP.BigQuery.Job("Count", {
131 * query: { query: "SELECT 1 AS n", useLegacySql: false },
132 * labels: { env: "prod" },
133 * });
134 * ```
135 *
136 * **Example:** Query into a destination table
137 * ```typescript
138 * const dataset = yield* GCP.BigQuery.Dataset("Analytics", {
139 * forceDestroy: true,
140 * });
141 * const table = yield* GCP.BigQuery.Table("Results", {
142 * datasetId: dataset.datasetId,
143 * schema: [{ name: "n", type: "INTEGER" }],
144 * });
145 * const load = yield* GCP.BigQuery.Job("Seed", {
146 * query: {
147 * query: "SELECT 1 AS n",
148 * useLegacySql: false,
149 * destinationTable: {
150 * projectId: table.project,
151 * datasetId: table.datasetId,
152 * tableId: table.tableId,
153 * },
154 * writeDisposition: "WRITE_TRUNCATE",
155 * },
156 * });
157 * ```
158 *
159 * @resource
160 * @category BigQuery
161 */
162 export const Job = Resource<Job>("GCP.BigQuery.Job");
163
164 export class JobNotResolved extends Data.TaggedError("GCP.BigQuery.JobNotResolved")<{
165 resourceName: string;
166 }> {}
167
168 export class JobNotDone extends Data.TaggedError("GCP.BigQuery.JobNotDone")<{
169 jobId: string;
170 state: string | undefined;
171 }> {}
172
173 const userLabels = (
174 labels: Record<string, string | undefined> | null | undefined,
175 ): Record<string, string> => stripInternalLabels(tagRecord(labels));
176
177 const resourceName = (project: string, jobId: string) => `projects/${project}/jobs/${jobId}`;
178
179 const normalizeLocation = (location: string | undefined, fallback: string) =>
180 (location ?? fallback).toUpperCase();
181
182 const toId = (id: string, jobId: string | undefined, existing?: string) =>
183 Effect.gen(function* () {
184 if (jobId !== undefined) return jobId;
185 if (existing !== undefined) return existing;
186 return yield* createPhysicalName({
187 id,
188 maxLength: MAX_JOB_ID_LENGTH,
189 lowercase: true,
190 });
191 });
192
193 const jobIdOf = (job: bigquery.Job | bigquery.JobListJobsItem, fallback: string) =>
194 job.jobReference?.jobId ?? fallback;
195
196 const projectOf = (job: bigquery.Job | bigquery.JobListJobsItem, fallback: string) =>
197 job.jobReference?.projectId ?? fallback;
198
199 const locationOf = (job: bigquery.Job | bigquery.JobListJobsItem, fallback: string) =>
200 job.jobReference?.location ?? fallback;
201
202 const stateOf = (job: bigquery.Job | bigquery.JobListJobsItem) => {
203 const full = job as bigquery.Job;
204 return full.status?.state ?? (job as bigquery.JobListJobsItem).state;
205 };
206
207 const toAttrs = (
208 job: bigquery.Job | bigquery.JobListJobsItem,
209 fallbackProject: string,
210 region: string,
211 ) => {
212 const jobId = jobIdOf(job, "");
213 const project = projectOf(job, fallbackProject);
214 const full = job as bigquery.Job;
215 return {
216 name: resourceName(project, jobId),
217 id: full.id ?? `${project}:${jobId}`,
218 jobId,
219 project,
220 location: locationOf(job, region),
221 labels: userLabels(job.configuration?.labels),
222 jobType: job.configuration?.jobType,
223 state: stateOf(job),
224 errorResult: full.status?.errorResult ?? (job as bigquery.JobListJobsItem).errorResult,
225 query: job.configuration?.query?.query,
226 selfLink: full.selfLink,
227 userEmail: full.user_email ?? (job as bigquery.JobListJobsItem).user_email,
228 creationTime: full.statistics?.creationTime ?? job.statistics?.creationTime,
229 startTime: full.statistics?.startTime ?? job.statistics?.startTime,
230 endTime: full.statistics?.endTime ?? job.statistics?.endTime,
231 };
232 };
233
234 const getByRef = (projectId: string, jobId: string, location: string) =>
235 bigquery
236 .getJobs({ projectId, jobId, location })
237 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
238
239 const waitUntilDone = (projectId: string, jobId: string, location: string) =>
240 getByRef(projectId, jobId, location).pipe(
241 Effect.flatMap((job) => {
242 if (job === undefined) return Effect.undefined;
243 const state = job.status?.state ?? "";
244 if (state === "DONE") return Effect.succeed(job);
245 return Effect.fail(new JobNotDone({ jobId, state }));
246 }),
247 Effect.retry({
248 while: (error) => error._tag === "GCP.BigQuery.JobNotDone",
249 times: 10,
250 schedule: Schedule.spaced("2 seconds"),
251 }),
252 );
253
254 const queryBody = (query: JobQuery): JobQuery => ({
255 ...query,
256 useLegacySql: query.useLegacySql ?? false,
257 });
258
259 const toJobBody = (
260 projectId: string,
261 jobId: string,
262 location: string,
263 news: JobProps,
264 labels: Record<string, string>,
265 ): bigquery.Job => ({
266 jobReference: { projectId, jobId, location },
267 configuration: {
268 labels,
269 query: news.query !== undefined ? queryBody(news.query) : undefined,
270 load: news.load,
271 copy: news.copy,
272 extract: news.extract,
273 jobTimeoutMs: news.jobTimeoutMs,
274 reservation: news.reservation,
275 },
276 });
277
278 const stable = (value: unknown): unknown => {
279 if (Array.isArray(value)) return value.map(stable);
280 if (value !== null && typeof value === "object") {
281 return Object.fromEntries(
282 Object.entries(value as Record<string, unknown>)
283 .filter(([, nested]) => nested !== undefined)
284 .sort(([left], [right]) => left.localeCompare(right))
285 .map(([key, nested]) => [key, stable(nested)]),
286 );
287 }
288 return value;
289 };
290
291 const jsonEqual = (left: unknown, right: unknown) =>
292 JSON.stringify(stable(left ?? null)) === JSON.stringify(stable(right ?? null));
293
294 const pickDefined = (news: Record<string, unknown>, observed: Record<string, unknown>) => {
295 const next: Record<string, unknown> = {};
296 const previous: Record<string, unknown> = {};
297 for (const [key, value] of Object.entries(news)) {
298 if (value !== undefined) {
299 next[key] = value;
300 previous[key] = observed[key];
301 }
302 }
303 return { news: next, observed: previous };
304 };
305
306 const nestedChanged = (
307 news: Record<string, unknown> | undefined,
308 olds: Record<string, unknown> | undefined,
309 hadOlds: boolean,
310 ) => {
311 if (news === undefined) return false;
312 if (olds === undefined) return hadOlds;
313 const picked = pickDefined(news, olds);
314 return !jsonEqual(picked.news, picked.observed);
315 };
316
317 export const JobProvider = () =>
318 Provider.succeed(Job, {
319 stables: ["name", "id", "jobId", "project", "location", "creationTime"],
320
321 diff: Effect.fn(function* ({ news, olds, output }) {
322 if (!isResolved(news)) return undefined;
323 const env = yield* GcpEnvironment.current;
324 const previousId = olds?.jobId ?? output?.jobId;
325 const jobIdChanged =
326 news.jobId !== undefined && previousId !== undefined && news.jobId !== previousId;
327
328 const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region);
329 const nextLocation = normalizeLocation(
330 news.location ?? olds?.location ?? output?.location,
331 env.region,
332 );
333 const locationChanged = output !== undefined && previousLocation !== nextLocation;
334
335 const labelsChanged =
336 news.labels !== undefined &&
337 output !== undefined &&
338 !jsonEqual(toLabels(news.labels), output.labels);
339
340 const timeoutChanged =
341 news.jobTimeoutMs !== undefined && (olds?.jobTimeoutMs ?? "") !== news.jobTimeoutMs;
342 const reservationChanged =
343 news.reservation !== undefined && (olds?.reservation ?? "") !== news.reservation;
344
345 const replaced =
346 jobIdChanged ||
347 locationChanged ||
348 labelsChanged ||
349 timeoutChanged ||
350 reservationChanged ||
351 nestedChanged(
352 news.query as Record<string, unknown> | undefined,
353 olds?.query as Record<string, unknown> | undefined,
354 olds !== undefined,
355 ) ||
356 nestedChanged(
357 news.load as Record<string, unknown> | undefined,
358 olds?.load as Record<string, unknown> | undefined,
359 olds !== undefined,
360 ) ||
361 nestedChanged(
362 news.copy as Record<string, unknown> | undefined,
363 olds?.copy as Record<string, unknown> | undefined,
364 olds !== undefined,
365 ) ||
366 nestedChanged(
367 news.extract as Record<string, unknown> | undefined,
368 olds?.extract as Record<string, unknown> | undefined,
369 olds !== undefined,
370 );
371
372 if (!replaced) return undefined;
373 // Job ids cannot be reused after delete (the API returns NotFound on
374 // insert of a retired id), so never delete-first.
375 return { action: "replace" as const, deleteFirst: false };
376 }),
377
378 read: Effect.fn(function* ({ id, olds, output }) {
379 const env = yield* GcpEnvironment.current;
380 const jobId = yield* toId(id, olds?.jobId, output?.jobId);
381 const location = olds?.location ?? output?.location ?? env.region;
382 const project = output?.project ?? env.project;
383 const existing = yield* getByRef(project, jobId, location);
384 if (existing === undefined) return undefined;
385 const attrs = toAttrs(existing, project, env.region);
386 return (yield* hasAlchemyLabels(id, tagRecord(existing.configuration?.labels)))
387 ? attrs
388 : Unowned(attrs);
389 }),
390
391 list: () =>
392 Effect.gen(function* () {
393 const env = yield* GcpEnvironment.current;
394 return yield* bigquery.listJobs
395 .pages({
396 projectId: env.project,
397 maxResults: 1000,
398 projection: "full",
399 allUsers: true,
400 })
401 .pipe(
402 Stream.flatMap((page) => Stream.fromIterable(page.jobs ?? [])),
403 Stream.filter((job) =>
404 Object.keys(job.configuration?.labels ?? {}).some((key) =>
405 key.startsWith("alchemy-"),
406 ),
407 ),
408 Stream.map((job) => toAttrs(job, env.project, env.region)),
409 Stream.runCollect,
410 Effect.map((chunk) => Array.from(chunk)),
411 Effect.catchTag("Forbidden", () =>
412 bigquery.listJobs
413 .pages({
414 projectId: env.project,
415 maxResults: 1000,
416 projection: "full",
417 })
418 .pipe(
419 Stream.flatMap((page) => Stream.fromIterable(page.jobs ?? [])),
420 Stream.filter((job) =>
421 Object.keys(job.configuration?.labels ?? {}).some((key) =>
422 key.startsWith("alchemy-"),
423 ),
424 ),
425 Stream.map((job) => toAttrs(job, env.project, env.region)),
426 Stream.runCollect,
427 Effect.map((chunk) => Array.from(chunk)),
428 ),
429 ),
430 Effect.catchTag("NotFound", () => Effect.succeed([] as ReturnType<typeof toAttrs>[])),
431 );
432 }),
433
434 reconcile: Effect.fn(function* ({ id, news, output }) {
435 const env = yield* GcpEnvironment.current;
436 const jobId = yield* toId(id, news.jobId, output?.jobId);
437 const location = news.location ?? output?.location ?? env.region;
438 const project = output?.project ?? env.project;
439 const name = resourceName(project, jobId);
440 const desiredLabels = {
441 ...toLabels(news.labels),
442 ...(yield* createInternalLabels(id)),
443 };
444
445 let current = yield* getByRef(project, jobId, location);
446
447 if (current === undefined) {
448 const created = yield* bigquery
449 .insertJobs({
450 projectId: project,
451 body: toJobBody(project, jobId, location, news, desiredLabels),
452 })
453 .pipe(Effect.catchTag("Conflict", () => getByRef(project, jobId, location)));
454 current = created ?? undefined;
455 }
456
457 if (current === undefined) {
458 return yield* new JobNotResolved({ resourceName: name });
459 }
460
461 const state = current.status?.state ?? "";
462 if (state !== "DONE") {
463 current = (yield* waitUntilDone(project, jobId, location)) ?? current;
464 }
465
466 return toAttrs(current, project, env.region);
467 }),
468
469 delete: Effect.fn(function* ({ output }) {
470 const state = output.state ?? "";
471 if (state === "PENDING" || state === "RUNNING") {
472 yield* bigquery
473 .cancelJobs({
474 projectId: output.project,
475 jobId: output.jobId,
476 location: output.location,
477 })
478 .pipe(
479 Effect.catchTags({
480 NotFound: () => Effect.void,
481 BadRequest: () => Effect.void,
482 }),
483 );
484 yield* waitUntilDone(output.project, output.jobId, output.location).pipe(
485 Effect.catchTag("GCP.BigQuery.JobNotDone", () => Effect.void),
486 );
487 }
488 yield* bigquery
489 .deleteJobs({
490 projectId: output.project,
491 jobId: output.jobId,
492 location: output.location,
493 })
494 .pipe(
495 Effect.retry({
496 while: (error) => error._tag === "Conflict" || error._tag === "BadRequest",
497 times: 8,
498 schedule: Schedule.spaced("2 seconds"),
499 }),
500 Effect.catchTag("NotFound", () => Effect.void),
501 );
502 }),
503 });