The best way to control Claude Code, Codex and more from any computer or phone

main
ts 497 lines 16 KB
Raw
1 import * as dataplex from "@distilled.cloud/gcp/dataplex_v1";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import { tagRecord } from "alchemy/Tags";
7 import * as Data from "effect/Data";
8 import * as Effect from "effect/Effect";
9 import * as Schedule from "effect/Schedule";
10 import { GcpEnvironment } from "../Environment.ts";
11 import { createInternalLabels, diffLabels, hasAlchemyLabels, toLabels } from "../Labels.ts";
12 import type { Providers } from "../Providers.ts";
13 import { waitForOperation } from "./operations.ts";
14 import {
15 DataplexNotResolved,
16 expandParent,
17 fingerprint,
18 hasAlchemyLabelMap,
19 isPendingState,
20 listChildResources,
21 listLakes,
22 listTasks,
23 normalizeLocation,
24 parseResourceName,
25 replaceIfChanged,
26 toPhysicalRfc1035,
27 userLabels,
28 } from "./shared.ts";
29
30 const DEFAULT_TRIGGER = "ON_DEMAND";
31
32 export type TaskTriggerSpec = {
33 /**
34 * Trigger type. Immutable — changing it replaces the task.
35 * @default "ON_DEMAND"
36 */
37 type?: dataplex.GoogleCloudDataplexV1TaskTriggerSpecTypeEnum | (string & {});
38 /** First run after this RFC3339 timestamp. */
39 startTime?: string;
40 /** Temporarily disable RECURRING execution. */
41 disabled?: boolean;
42 /** Retry attempts before aborting. */
43 maxRetries?: number;
44 /** Cron schedule. Required for `RECURRING`. */
45 schedule?: string;
46 };
47
48 export type TaskExecutionSpec = {
49 /** Arguments interpolated before passing to the driver. */
50 args?: Record<string, string>;
51 /**
52 * Service account used to execute the task. If omitted, the project
53 * Compute Engine default service account is used.
54 */
55 serviceAccount?: string;
56 /** Project in which jobs run. Defaults to the lake project. */
57 project?: string;
58 /** Maximum job duration (e.g. `"3600s"`). */
59 maxJobExecutionLifetime?: string;
60 /** Cloud KMS key used to encrypt job data. */
61 kmsKey?: string;
62 };
63
64 export type TaskInfrastructureSpec = {
65 /** Dataproc Serverless executor counts. */
66 batch?: {
67 executorsCount?: number;
68 maxExecutorsCount?: number;
69 };
70 /** Container image runtime. */
71 containerImage?: {
72 image?: string;
73 javaJars?: string[];
74 pythonPackages?: string[];
75 properties?: Record<string, string>;
76 };
77 /** VPC network used to run the job. */
78 vpcNetwork?: {
79 network?: string;
80 subNetwork?: string;
81 networkTags?: string[];
82 };
83 };
84
85 export type TaskSparkConfig = {
86 /** Cloud Storage URI of the jar containing the main class. */
87 mainJarFileUri?: string;
88 /** Driver main class. */
89 mainClass?: string;
90 /** Cloud Storage URI of the main Python file. */
91 pythonScriptFile?: string;
92 /** Cloud Storage URI of a SQL query file. */
93 sqlScriptFile?: string;
94 /** Inline SQL query text. */
95 sqlScript?: string;
96 /** Working-directory files. */
97 fileUris?: string[];
98 /** Archives extracted into the working directory. */
99 archiveUris?: string[];
100 /** Infrastructure used to run the Spark job. */
101 infrastructureSpec?: TaskInfrastructureSpec;
102 };
103
104 export type TaskNotebookConfig = {
105 /** Path to the input notebook. */
106 notebook: string;
107 /** Infrastructure used to run the notebook. */
108 infrastructureSpec?: TaskInfrastructureSpec;
109 /** Working-directory files. */
110 fileUris?: string[];
111 /** Archives extracted into the working directory. */
112 archiveUris?: string[];
113 };
114
115 export type LakesTaskProps = {
116 /**
117 * Parent lake. Full name
118 * `projects/{project}/locations/{location}/lakes/{lake}` or the lake id
119 * (combined with `location`). Immutable — changing it replaces the task.
120 */
121 lake: string;
122 /**
123 * Region used when `lake` is a bare id.
124 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
125 */
126 location?: string;
127 /**
128 * Task id. Immutable — changing it replaces the task.
129 */
130 taskId?: string;
131 /**
132 * Human-readable description.
133 */
134 description?: string;
135 /**
136 * User-friendly display name.
137 */
138 displayName?: string;
139 /**
140 * User labels. Alchemy ownership labels are merged in automatically.
141 */
142 labels?: Record<string, string>;
143 /**
144 * How often and when the task runs.
145 * @default { type: "ON_DEMAND" }
146 */
147 triggerSpec?: TaskTriggerSpec;
148 /**
149 * How the task is executed (service account, args, encryption).
150 */
151 executionSpec?: TaskExecutionSpec;
152 /**
153 * Spark workload. Mutually exclusive with `notebook`.
154 */
155 spark?: TaskSparkConfig;
156 /**
157 * Notebook workload. Mutually exclusive with `spark`.
158 */
159 notebook?: TaskNotebookConfig;
160 };
161
162 export type LakesTask = Resource<
163 "GCP.Dataplex.LakesTask",
164 LakesTaskProps,
165 {
166 /** Full resource name `.../lakes/{lake}/tasks/{task}`. */
167 name: string;
168 /** Task id. */
169 taskId: string;
170 /** Parent lake resource name. */
171 lake: string;
172 /** Project id. */
173 project: string;
174 /** Location id. */
175 location: string;
176 /** Description. */
177 description: string | undefined;
178 /** User-friendly display name. */
179 displayName: string | undefined;
180 /** User labels (Alchemy ownership labels stripped). */
181 labels: Record<string, string>;
182 /** Trigger type. */
183 triggerType: string | undefined;
184 /** Execution service account. */
185 serviceAccount: string | undefined;
186 /** Lifecycle state. */
187 state: string | undefined;
188 /** Server-assigned uid. */
189 uid: string | undefined;
190 /** RFC3339 creation timestamp. */
191 createTime: string | undefined;
192 /** RFC3339 last-update timestamp. */
193 updateTime: string | undefined;
194 },
195 never,
196 Providers
197 >;
198
199 /**
200 * A Dataplex lake task — a Spark or notebook job scheduled against a lake.
201 *
202 * Changing `lake`, `taskId`, `location`, or trigger type replaces the
203 * task. Description, labels, execution spec, and Spark/notebook config
204 * update in place.
205 *
206 * ### Creating a Task
207 * **Example:** On-demand Spark SQL
208 * ```typescript
209 * const task = yield* GCP.Dataplex.LakesTask("SelectOne", {
210 * lake: lake.name,
211 * executionSpec: { serviceAccount },
212 * spark: { sqlScript: "SELECT 1" },
213 * });
214 * ```
215 *
216 * **Example:** Recurring notebook
217 * ```typescript
218 * const task = yield* GCP.Dataplex.LakesTask("Nightly", {
219 * lake: lake.name,
220 * triggerSpec: { type: "RECURRING", schedule: "0 2 * * *" },
221 * executionSpec: { serviceAccount },
222 * notebook: { notebook: "gs://bucket/job.ipynb" },
223 * });
224 * ```
225 *
226 * @resource
227 * @category Dataplex
228 */
229 export const LakesTask = Resource<LakesTask>("GCP.Dataplex.LakesTask");
230
231 export class LakesTaskNotResolved extends Data.TaggedError("GCP.Dataplex.LakesTaskNotResolved")<{
232 resourceName: string;
233 }> {}
234
235 export class LakesTaskStillExists extends Data.TaggedError("GCP.Dataplex.LakesTaskStillExists")<{
236 resourceName: string;
237 }> {}
238
239 const lakeOf = (lake: string, project: string, location: string) =>
240 expandParent(lake, project, location, "lakes");
241
242 const resourceName = (lake: string, taskId: string) => `${lake}/tasks/${taskId}`;
243
244 const triggerBody = (
245 spec: TaskTriggerSpec | undefined,
246 ): dataplex.GoogleCloudDataplexV1TaskTriggerSpec => ({
247 type: (spec?.type ?? DEFAULT_TRIGGER).toUpperCase(),
248 startTime: spec?.startTime,
249 disabled: spec?.disabled,
250 maxRetries: spec?.maxRetries,
251 schedule: spec?.schedule,
252 });
253
254 const executionBody = (
255 spec: TaskExecutionSpec | undefined,
256 ): dataplex.GoogleCloudDataplexV1TaskExecutionSpec => ({
257 args: spec?.args,
258 serviceAccount: spec?.serviceAccount,
259 project: spec?.project,
260 maxJobExecutionLifetime: spec?.maxJobExecutionLifetime,
261 kmsKey: spec?.kmsKey,
262 });
263
264 const toAttrs = (task: dataplex.GoogleCloudDataplexV1Task, project: string) => {
265 const name = task.name ?? "";
266 const parsed = parseResourceName(name, "tasks");
267 return {
268 name,
269 taskId: parsed.id,
270 lake: parsed.parent,
271 project: parsed.project || project,
272 location: parsed.location,
273 description: task.description,
274 displayName: task.displayName,
275 labels: userLabels(task.labels),
276 triggerType: task.triggerSpec?.type,
277 serviceAccount: task.executionSpec?.serviceAccount,
278 state: task.state,
279 uid: task.uid,
280 createTime: task.createTime,
281 updateTime: task.updateTime,
282 };
283 };
284
285 const getByName = (name: string) =>
286 name.length === 0 || name.includes("//")
287 ? Effect.undefined
288 : dataplex
289 .getProjectsLocationsLakesTasks({ name })
290 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
291
292 const waitUntilReady = (name: string) =>
293 getByName(name).pipe(
294 Effect.filterOrFail(
295 (task): task is dataplex.GoogleCloudDataplexV1Task => task !== undefined,
296 () => new LakesTaskNotResolved({ resourceName: name }),
297 ),
298 Effect.filterOrFail(
299 (task) => !isPendingState(task.state),
300 () => new DataplexNotResolved({ resourceName: name }),
301 ),
302 Effect.retry({
303 while: (error) =>
304 error._tag === "GCP.Dataplex.LakesTaskNotResolved" ||
305 error._tag === "GCP.Dataplex.NotResolved" ||
306 error._tag === "TooManyRequests",
307 times: 10,
308 schedule: Schedule.spaced("10 seconds"),
309 }),
310 );
311
312 const waitUntilGone = (name: string) =>
313 getByName(name).pipe(
314 Effect.flatMap((task) =>
315 task === undefined
316 ? Effect.void
317 : Effect.fail(new LakesTaskStillExists({ resourceName: name })),
318 ),
319 Effect.retry({
320 while: (error) => error._tag === "GCP.Dataplex.LakesTaskStillExists",
321 times: 10,
322 schedule: Schedule.spaced("3 seconds"),
323 }),
324 );
325
326 export const LakesTaskProvider = () =>
327 Provider.succeed(LakesTask, {
328 stables: ["name", "taskId", "lake", "project", "location", "uid", "createTime"],
329
330 diff: Effect.fn(function* ({ news, olds, output }) {
331 if (!isResolved(news)) return undefined;
332 const env = yield* GcpEnvironment.current;
333 const previousId = olds?.taskId ?? output?.taskId;
334 const nextId = news.taskId ?? previousId;
335 const previousLake = olds?.lake ?? output?.lake;
336 const nextLake = news.lake ?? previousLake;
337 const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region);
338 const nextLocation = normalizeLocation(
339 news.location ?? olds?.location ?? output?.location,
340 env.region,
341 );
342 const previousTrigger = (
343 olds?.triggerSpec?.type ??
344 output?.triggerType ??
345 DEFAULT_TRIGGER
346 ).toUpperCase();
347 const nextTrigger = (news.triggerSpec?.type ?? previousTrigger).toUpperCase();
348 if (
349 replaceIfChanged(previousId, nextId) ||
350 replaceIfChanged(previousLake, nextLake) ||
351 (output !== undefined && previousLocation !== nextLocation) ||
352 previousTrigger !== nextTrigger
353 ) {
354 return {
355 action: "replace" as const,
356 deleteFirst:
357 previousLake === nextLake &&
358 previousLocation === nextLocation &&
359 previousId !== undefined &&
360 nextId === previousId,
361 };
362 }
363 return undefined;
364 }),
365
366 read: Effect.fn(function* ({ id, olds, output }) {
367 const env = yield* GcpEnvironment.current;
368 const location = normalizeLocation(olds?.location ?? output?.location, env.region);
369 const lake = lakeOf(olds?.lake ?? output?.lake ?? "", env.project, location);
370 const taskId = yield* toPhysicalRfc1035(id, olds?.taskId, output?.taskId);
371 const name = output?.name ?? resourceName(lake, taskId);
372 const existing = yield* getByName(name);
373 if (existing === undefined) return undefined;
374 const attrs = toAttrs(existing, env.project);
375 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
376 }),
377
378 list: () =>
379 Effect.gen(function* () {
380 const env = yield* GcpEnvironment.current;
381 const lakes = yield* listLakes(env.project, env.region);
382 const tasks = yield* listChildResources(lakes, listTasks);
383 return tasks
384 .filter((task) => hasAlchemyLabelMap(task.labels))
385 .map((task) => toAttrs(task, env.project));
386 }),
387
388 reconcile: Effect.fn(function* ({ id, news, output }) {
389 const env = yield* GcpEnvironment.current;
390 const location = normalizeLocation(news.location ?? output?.location, env.region);
391 const lake = lakeOf(news.lake, env.project, location);
392 const taskId = yield* toPhysicalRfc1035(id, news.taskId, output?.taskId);
393 const name = output?.name ?? resourceName(lake, taskId);
394 const desiredLabels = {
395 ...toLabels(news.labels),
396 ...(yield* createInternalLabels(id)),
397 };
398 const triggerSpec = triggerBody(news.triggerSpec);
399 const executionSpec = executionBody(news.executionSpec);
400
401 let current = yield* getByName(name);
402
403 if (current === undefined) {
404 const created = yield* dataplex
405 .createProjectsLocationsLakesTasks({
406 parent: lake,
407 taskId,
408 body: {
409 description: news.description,
410 displayName: news.displayName,
411 labels: desiredLabels,
412 triggerSpec,
413 executionSpec,
414 spark: news.spark,
415 notebook: news.notebook,
416 },
417 })
418 .pipe(Effect.catchTag("Conflict", () => Effect.undefined));
419 if (created !== undefined) {
420 yield* waitForOperation(created);
421 }
422 current = yield* waitUntilReady(name);
423 }
424
425 if (current === undefined) {
426 return yield* new LakesTaskNotResolved({ resourceName: name });
427 }
428
429 const observedLabels = tagRecord(current.labels);
430 const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
431 const labelsChanged = upsert.length > 0 || removed.length > 0;
432 const descriptionChanged = (current.description ?? "") !== (news.description ?? "");
433 const displayNameChanged = (current.displayName ?? "") !== (news.displayName ?? "");
434 const triggerChanged = fingerprint(current.triggerSpec) !== fingerprint(triggerSpec);
435 const executionChanged = fingerprint(current.executionSpec) !== fingerprint(executionSpec);
436 const sparkChanged = fingerprint(current.spark) !== fingerprint(news.spark);
437 const notebookChanged = fingerprint(current.notebook) !== fingerprint(news.notebook);
438
439 if (
440 labelsChanged ||
441 descriptionChanged ||
442 displayNameChanged ||
443 triggerChanged ||
444 executionChanged ||
445 sparkChanged ||
446 notebookChanged
447 ) {
448 const updateMask = [
449 labelsChanged ? "labels" : undefined,
450 descriptionChanged ? "description" : undefined,
451 displayNameChanged ? "display_name" : undefined,
452 triggerChanged ? "trigger_spec" : undefined,
453 executionChanged ? "execution_spec" : undefined,
454 sparkChanged ? "spark" : undefined,
455 notebookChanged ? "notebook" : undefined,
456 ].filter((field): field is string => field !== undefined);
457 const operation = yield* dataplex.patchProjectsLocationsLakesTasks({
458 name: current.name ?? name,
459 updateMask: updateMask.join(","),
460 body: {
461 description: news.description,
462 displayName: news.displayName,
463 labels: desiredLabels,
464 triggerSpec,
465 executionSpec,
466 spark: news.spark,
467 notebook: news.notebook,
468 },
469 });
470 yield* waitForOperation(operation);
471 current = yield* waitUntilReady(current.name ?? name);
472 }
473
474 return toAttrs(current, env.project);
475 }),
476
477 delete: Effect.fn(function* ({ output }) {
478 if (!output.name || output.name.includes("//")) return;
479 const operation = yield* dataplex
480 .deleteProjectsLocationsLakesTasks({ name: output.name })
481 .pipe(
482 Effect.retry({
483 while: (error) => error._tag === "Conflict",
484 times: 8,
485 schedule: Schedule.spaced("2 seconds"),
486 }),
487 Effect.catchTags({
488 NotFound: () => Effect.undefined,
489 BadRequest: () => Effect.undefined,
490 }),
491 );
492 if (operation !== undefined) {
493 yield* waitForOperation(operation, { notFoundOk: true });
494 }
495 yield* waitUntilGone(output.name);
496 }),
497 });