The best way to control Claude Code, Codex and more from any computer or phone

main
ts 327 lines 10.6 KB
Raw
1 import * as dataproc from "@distilled.cloud/gcp/dataproc_v1";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import { tagRecord } from "alchemy/Tags";
7 import * as Data from "effect/Data";
8 import * as Effect from "effect/Effect";
9 import { GcpEnvironment } from "../Environment.ts";
10 import { createInternalLabels, diffLabels, hasAlchemyLabels, toLabels } from "../Labels.ts";
11 import type { Providers } from "../Providers.ts";
12 import {
13 collectPages,
14 LIST_LOCATIONS,
15 MAX_MANAGED_CLUSTER_PREFIX_LENGTH,
16 MAX_POLICY_ID_LENGTH,
17 defaultWorkflowJobs,
18 defaultWorkflowPlacement,
19 emptyOnMissing,
20 fingerprint,
21 hasAlchemyLabelMap,
22 locationParent,
23 normalizeLocation,
24 parseResourceName,
25 rfc1035,
26 toPhysicalId,
27 userLabels,
28 waitUntilGone,
29 } from "./internal.ts";
30
31 export type WorkflowTemplateProps = {
32 /**
33 * Template id (the `{template}` segment of
34 * `projects/{project}/locations/{location}/workflowTemplates/{template}`).
35 * If omitted, a unique RFC1035 name is generated. Immutable — changing
36 * it replaces the template.
37 */
38 templateId?: string;
39 /**
40 * Location (`us-central1`, …). Immutable — changing it replaces the
41 * template. `US-CENTRAL1` is accepted and normalized to `us-central1`.
42 * @default the stack's GCP region (`GCP.Region`, profile region, `us-central1`)
43 */
44 location?: string;
45 /**
46 * User labels. Alchemy ownership labels are merged in automatically.
47 */
48 labels?: Record<string, string>;
49 /**
50 * Directed acyclic graph of jobs. Defaults to a SparkPi example step.
51 */
52 jobs?: dataproc.OrderedJobList;
53 /**
54 * Target cluster (managed cluster or selector). Defaults to a 1-master
55 * 2-worker managed cluster template; creating the workflow does not
56 * provision the cluster until the template is instantiated.
57 */
58 placement?: dataproc.WorkflowTemplatePlacement;
59 /**
60 * DAG timeout (JSON Duration, 10m-24h).
61 */
62 dagTimeout?: string;
63 /**
64 * Cloud KMS key for encrypting job arguments.
65 */
66 kmsKey?: string;
67 /**
68 * Template parameters substituted at instantiate time.
69 */
70 parameters?: dataproc.TemplateParameterList;
71 };
72
73 export type WorkflowTemplate = Resource<
74 "GCP.Dataproc.WorkflowTemplate",
75 WorkflowTemplateProps,
76 {
77 /** Full resource name `projects/{project}/locations/{location}/workflowTemplates/{template}`. */
78 name: string;
79 /** Template id (last path segment). */
80 templateId: string;
81 /** Project id. */
82 project: string;
83 /** Location id. */
84 location: string;
85 /** User labels (Alchemy ownership labels stripped). */
86 labels: Record<string, string>;
87 /** Server version used for optimistic updates. */
88 version: number | undefined;
89 /** DAG timeout. */
90 dagTimeout: string | undefined;
91 /** RFC3339 creation timestamp. */
92 createTime: string | undefined;
93 /** RFC3339 last-update timestamp. */
94 updateTime: string | undefined;
95 },
96 never,
97 Providers
98 >;
99
100 /**
101 * A Dataproc workflow template (locations API).
102 *
103 * Stores a DAG of jobs and a cluster placement. Instantiating the template
104 * (out of band) is what actually creates a cluster and runs jobs.
105 *
106 * Changing `templateId` or `location` replaces the template. Jobs,
107 * placement, labels, and timeout update in place (optimistic version).
108 *
109 * ### Creating a Template
110 * **Example:** Generated name
111 * ```typescript
112 * const template = yield* GCP.Dataproc.WorkflowTemplate("Nightly", {});
113 * ```
114 *
115 * **Example:** Explicit jobs
116 * ```typescript
117 * const template = yield* GCP.Dataproc.WorkflowTemplate("Nightly", {
118 * templateId: "nightly-etl",
119 * location: "us-central1",
120 * labels: { env: "prod" },
121 * jobs: [{
122 * stepId: "spark-pi",
123 * sparkJob: {
124 * mainClass: "org.apache.spark.examples.SparkPi",
125 * jarFileUris: ["file:///usr/lib/spark/examples/jars/spark-examples.jar"],
126 * },
127 * }],
128 * });
129 * ```
130 *
131 * @resource
132 * @category Dataproc
133 */
134 export const WorkflowTemplate = Resource<WorkflowTemplate>("GCP.Dataproc.WorkflowTemplate");
135
136 export class WorkflowTemplateNotResolved extends Data.TaggedError(
137 "GCP.Dataproc.WorkflowTemplateNotResolved",
138 )<{
139 resourceName: string;
140 }> {}
141
142 const resourceName = (project: string, location: string, templateId: string) =>
143 `${locationParent(project, location)}/workflowTemplates/${templateId}`;
144
145 const toAttrs = (template: dataproc.WorkflowTemplate, project: string, location: string) => {
146 const name = template.name ?? "";
147 const parsed = parseResourceName(name, "workflowTemplates", location);
148 return {
149 name,
150 templateId: template.id ?? parsed.id,
151 project: parsed.project || project,
152 location: parsed.location || location,
153 labels: userLabels(template.labels),
154 version: template.version,
155 dagTimeout: template.dagTimeout,
156 createTime: template.createTime,
157 updateTime: template.updateTime,
158 };
159 };
160
161 const desiredBody = (
162 news: WorkflowTemplateProps,
163 templateId: string,
164 name: string,
165 desiredLabels: Record<string, string>,
166 version?: number,
167 ): dataproc.WorkflowTemplate => ({
168 id: templateId,
169 name,
170 labels: desiredLabels,
171 version,
172 jobs: news.jobs ?? defaultWorkflowJobs(),
173 placement:
174 news.placement ??
175 defaultWorkflowPlacement(
176 rfc1035(`${templateId}-c`, MAX_MANAGED_CLUSTER_PREFIX_LENGTH, "cluster"),
177 ),
178 dagTimeout: news.dagTimeout,
179 parameters: news.parameters,
180 encryptionConfig: news.kmsKey ? { kmsKey: news.kmsKey } : undefined,
181 });
182
183 const getByName = (name: string) =>
184 dataproc
185 .getProjectsLocationsWorkflowTemplates({ name })
186 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
187
188 const listLocation = (project: string, location: string) =>
189 emptyOnMissing(
190 collectPages(
191 dataproc.listProjectsLocationsWorkflowTemplates.pages({
192 parent: locationParent(project, location),
193 pageSize: 1000,
194 }),
195 (page) => page.templates,
196 ).pipe(
197 Effect.map((items) =>
198 items
199 .filter((template) => hasAlchemyLabelMap(template.labels))
200 .map((template) => toAttrs(template, project, location)),
201 ),
202 ),
203 );
204
205 const templateChanged = (current: dataproc.WorkflowTemplate, desired: dataproc.WorkflowTemplate) =>
206 fingerprint(current.jobs) !== fingerprint(desired.jobs) ||
207 fingerprint(current.placement) !== fingerprint(desired.placement) ||
208 (current.dagTimeout ?? "") !== (desired.dagTimeout ?? "") ||
209 fingerprint(current.parameters) !== fingerprint(desired.parameters) ||
210 (current.encryptionConfig?.kmsKey ?? "") !== (desired.encryptionConfig?.kmsKey ?? "");
211
212 export const WorkflowTemplateProvider = () =>
213 Provider.succeed(WorkflowTemplate, {
214 stables: ["name", "templateId", "project", "location", "createTime"],
215
216 diff: Effect.fn(function* ({ news, olds, output }) {
217 if (!isResolved(news)) return undefined;
218 const env = yield* GcpEnvironment.current;
219 const previousId = olds?.templateId ?? output?.templateId;
220 const nextId = news.templateId ?? previousId;
221 const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region);
222 const nextLocation = normalizeLocation(
223 news.location ?? olds?.location ?? output?.location,
224 env.region,
225 );
226 if (
227 (previousId !== undefined && nextId !== undefined && previousId !== nextId) ||
228 (output !== undefined && previousLocation !== nextLocation)
229 ) {
230 return {
231 action: "replace" as const,
232 deleteFirst:
233 previousLocation === nextLocation && previousId !== undefined && nextId === previousId,
234 };
235 }
236 return undefined;
237 }),
238
239 read: Effect.fn(function* ({ id, olds, output }) {
240 const env = yield* GcpEnvironment.current;
241 const location = normalizeLocation(olds?.location ?? output?.location, env.region);
242 const templateId = yield* toPhysicalId(
243 id,
244 olds?.templateId,
245 output?.templateId,
246 MAX_POLICY_ID_LENGTH,
247 "template",
248 );
249 const name = output?.name ?? resourceName(env.project, location, templateId);
250 const existing = yield* getByName(name);
251 if (existing === undefined) return undefined;
252 const attrs = toAttrs(existing, env.project, location);
253 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
254 }),
255
256 list: () =>
257 Effect.gen(function* () {
258 const env = yield* GcpEnvironment.current;
259 const pages = yield* Effect.forEach(
260 [...new Set([env.region, ...LIST_LOCATIONS])],
261 (location) => listLocation(env.project, location),
262 { concurrency: 4 },
263 );
264 return pages.flat();
265 }),
266
267 reconcile: Effect.fn(function* ({ id, news, output }) {
268 const env = yield* GcpEnvironment.current;
269 const location = normalizeLocation(news.location ?? output?.location, env.region);
270 const templateId = yield* toPhysicalId(
271 id,
272 news.templateId,
273 output?.templateId,
274 MAX_POLICY_ID_LENGTH,
275 "template",
276 );
277 const name = resourceName(env.project, location, templateId);
278 const desiredLabels = {
279 ...toLabels(news.labels),
280 ...(yield* createInternalLabels(id)),
281 };
282
283 let current = yield* getByName(name);
284
285 if (current === undefined) {
286 const created = yield* dataproc
287 .createProjectsLocationsWorkflowTemplates({
288 parent: locationParent(env.project, location),
289 body: desiredBody(news, templateId, name, desiredLabels),
290 })
291 .pipe(Effect.catchTag("Conflict", () => getByName(name)));
292 current = created ?? undefined;
293 }
294
295 if (current === undefined) {
296 return yield* new WorkflowTemplateNotResolved({ resourceName: name });
297 }
298
299 const desired = desiredBody(
300 news,
301 templateId,
302 current.name ?? name,
303 desiredLabels,
304 current.version,
305 );
306 const observedLabels = tagRecord(current.labels);
307 const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
308 const labelsChanged = upsert.length > 0 || removed.length > 0;
309
310 if (labelsChanged || templateChanged(current, desired)) {
311 current = yield* dataproc.updateProjectsLocationsWorkflowTemplates({
312 name: current.name ?? name,
313 body: desired,
314 });
315 }
316
317 return toAttrs(current, env.project, location);
318 }),
319
320 delete: Effect.fn(function* ({ output }) {
321 if (!output.name) return;
322 yield* dataproc
323 .deleteProjectsLocationsWorkflowTemplates({ name: output.name })
324 .pipe(Effect.catchTag("NotFound", () => Effect.void));
325 yield* waitUntilGone(getByName(output.name), output.name);
326 }),
327 });