@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 351 lines 10.8 KB
Raw
1 import * as dataproc from "@distilled.cloud/gcp/dataproc_v1";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import { tagRecord } from "alchemy/Tags";
7 import * as Data from "effect/Data";
8 import * as Effect from "effect/Effect";
9 import * as Schedule from "effect/Schedule";
10 import { GcpEnvironment } from "../Environment.ts";
11 import { createInternalLabels, hasAlchemyLabels, toLabels } from "../Labels.ts";
12 import type { Providers } from "../Providers.ts";
13 import {
14 collectPages,
15 LIST_LOCATIONS,
16 MAX_POLLS,
17 MAX_WORKLOAD_ID_LENGTH,
18 defaultSparkBatch,
19 emptyOnMissing,
20 hasAlchemyLabelMap,
21 locationParent,
22 normalizeLocation,
23 parseResourceName,
24 toPhysicalId,
25 userLabels,
26 waitForOperation,
27 waitUntilExists,
28 waitUntilGone,
29 } from "./internal.ts";
30
31 const TERMINAL = new Set(["SUCCEEDED", "FAILED", "CANCELLED", "CANCELLING"]);
32
33 export type BatchProps = {
34 /**
35 * Batch id (the `{batch}` segment of
36 * `projects/{project}/locations/{location}/batches/{batch}`). If omitted,
37 * a unique RFC1035 name is generated. 4-63 characters. Immutable —
38 * changing it replaces the batch.
39 */
40 batchId?: string;
41 /**
42 * Location (`us-central1`, …). Immutable — changing it replaces the
43 * batch. `US-CENTRAL1` is accepted and normalized to `us-central1`.
44 * @default the stack's GCP region (`GCP.Region`, profile region, `us-central1`)
45 */
46 location?: string;
47 /**
48 * User labels. Alchemy ownership labels are merged in automatically.
49 */
50 labels?: Record<string, string>;
51 /**
52 * Spark batch config. Defaults to SparkPi from the runtime image when
53 * no other workload kind is set.
54 */
55 sparkBatch?: dataproc.SparkBatch;
56 /**
57 * PySpark batch config.
58 */
59 pysparkBatch?: dataproc.PySparkBatch;
60 /**
61 * SparkR batch config.
62 */
63 sparkRBatch?: dataproc.SparkRBatch;
64 /**
65 * Spark SQL batch config.
66 */
67 sparkSqlBatch?: dataproc.SparkSqlBatch;
68 /**
69 * PySpark notebook batch config.
70 */
71 pysparkNotebookBatch?: dataproc.PySparkNotebookBatch;
72 /**
73 * Runtime configuration (version, properties, container image).
74 */
75 runtimeConfig?: dataproc.RuntimeConfig;
76 /**
77 * Environment configuration (execution and peripherals).
78 */
79 environmentConfig?: dataproc.EnvironmentConfig;
80 };
81
82 export type Batch = Resource<
83 "GCP.Dataproc.Batch",
84 BatchProps,
85 {
86 /** Full resource name `projects/{project}/locations/{location}/batches/{batch}`. */
87 name: string;
88 /** Batch id (last path segment). */
89 batchId: string;
90 /** Project id. */
91 project: string;
92 /** Location id. */
93 location: string;
94 /** User labels (Alchemy ownership labels stripped). */
95 labels: Record<string, string>;
96 /** Server-reported state (`PENDING`, `RUNNING`, `SUCCEEDED`, …). */
97 state: string | undefined;
98 /** Extra status text, if any. */
99 stateMessage: string | undefined;
100 /** Server-assigned uuid. */
101 uuid: string | undefined;
102 /** Associated long-running operation. */
103 operation: string | undefined;
104 /** Creator email. */
105 creator: string | undefined;
106 /** RFC3339 creation timestamp. */
107 createTime: string | undefined;
108 },
109 never,
110 Providers
111 >;
112
113 /**
114 * A Dataproc serverless batch workload.
115 *
116 * Batches are immutable after create. Changing identity or config
117 * replaces the batch (re-runs the workload). The resource exists as soon
118 * as the create operation returns; Spark completion is not required.
119 *
120 * ### Creating a Batch
121 * **Example:** SparkPi
122 * ```typescript
123 * const batch = yield* GCP.Dataproc.Batch("Pi", {
124 * sparkBatch: {
125 * mainClass: "org.apache.spark.examples.SparkPi",
126 * jarFileUris: ["file:///usr/lib/spark/examples/jars/spark-examples.jar"],
127 * args: ["1"],
128 * },
129 * });
130 * ```
131 *
132 * **Example:** PySpark
133 * ```typescript
134 * const batch = yield* GCP.Dataproc.Batch("Etl", {
135 * pysparkBatch: { mainPythonFileUri: "gs://bucket/job.py" },
136 * labels: { env: "prod" },
137 * });
138 * ```
139 *
140 * @resource
141 * @category Dataproc
142 */
143 export const Batch = Resource<Batch>("GCP.Dataproc.Batch");
144
145 export class BatchNotResolved extends Data.TaggedError("GCP.Dataproc.BatchNotResolved")<{
146 resourceName: string;
147 }> {}
148
149 export class BatchNotTerminal extends Data.TaggedError("GCP.Dataproc.BatchNotTerminal")<{
150 resourceName: string;
151 state: string | undefined;
152 }> {}
153
154 const resourceName = (project: string, location: string, batchId: string) =>
155 `${locationParent(project, location)}/batches/${batchId}`;
156
157 const defaultSpark = (news: BatchProps): dataproc.SparkBatch | undefined => {
158 if (
159 news.pysparkBatch !== undefined ||
160 news.sparkRBatch !== undefined ||
161 news.sparkSqlBatch !== undefined ||
162 news.pysparkNotebookBatch !== undefined
163 ) {
164 return news.sparkBatch;
165 }
166 return news.sparkBatch ?? defaultSparkBatch();
167 };
168
169 const toAttrs = (batch: dataproc.Batch, project: string, location: string) => {
170 const name = batch.name ?? "";
171 const parsed = parseResourceName(name, "batches", location);
172 return {
173 name,
174 batchId: parsed.id,
175 project: parsed.project || project,
176 location: parsed.location || location,
177 labels: userLabels(batch.labels),
178 state: batch.state,
179 stateMessage: batch.stateMessage,
180 uuid: batch.uuid,
181 operation: batch.operation,
182 creator: batch.creator,
183 createTime: batch.createTime,
184 };
185 };
186
187 const desiredBody = (news: BatchProps, desiredLabels: Record<string, string>): dataproc.Batch => ({
188 labels: desiredLabels,
189 sparkBatch: defaultSpark(news),
190 pysparkBatch: news.pysparkBatch,
191 sparkRBatch: news.sparkRBatch,
192 sparkSqlBatch: news.sparkSqlBatch,
193 pysparkNotebookBatch: news.pysparkNotebookBatch,
194 runtimeConfig: news.runtimeConfig,
195 environmentConfig: news.environmentConfig ?? {
196 executionConfig: { ttl: "600s" },
197 },
198 });
199
200 const getByName = (name: string) =>
201 dataproc
202 .getProjectsLocationsBatches({ name })
203 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
204
205 const waitUntilTerminal = (name: string) =>
206 getByName(name).pipe(
207 Effect.flatMap((batch) =>
208 batch === undefined
209 ? Effect.undefined
210 : TERMINAL.has(batch.state ?? "")
211 ? Effect.succeed(batch)
212 : Effect.fail(
213 new BatchNotTerminal({
214 resourceName: name,
215 state: batch.state,
216 }),
217 ),
218 ),
219 Effect.retry({
220 while: (error) => error._tag === "GCP.Dataproc.BatchNotTerminal",
221 times: MAX_POLLS,
222 schedule: Schedule.spaced("5 seconds"),
223 }),
224 );
225
226 const listLocation = (project: string, location: string) =>
227 emptyOnMissing(
228 collectPages(
229 dataproc.listProjectsLocationsBatches.pages({
230 parent: locationParent(project, location),
231 pageSize: 1000,
232 }),
233 (page) => page.batches,
234 ).pipe(
235 Effect.map((items) =>
236 items
237 .filter((batch) => hasAlchemyLabelMap(batch.labels))
238 .map((batch) => toAttrs(batch, project, location)),
239 ),
240 ),
241 );
242
243 export const BatchProvider = () =>
244 Provider.succeed(Batch, {
245 stables: ["name", "batchId", "project", "location", "uuid", "createTime"],
246
247 diff: Effect.fn(function* ({ news, olds, output }) {
248 if (!isResolved(news)) return undefined;
249 const env = yield* GcpEnvironment.current;
250 const previousId = olds?.batchId ?? output?.batchId;
251 const nextId = news.batchId ?? previousId;
252 const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region);
253 const nextLocation = normalizeLocation(
254 news.location ?? olds?.location ?? output?.location,
255 env.region,
256 );
257 if (
258 (previousId !== undefined && nextId !== undefined && previousId !== nextId) ||
259 (output !== undefined && previousLocation !== nextLocation)
260 ) {
261 return {
262 action: "replace" as const,
263 deleteFirst:
264 previousLocation === nextLocation && previousId !== undefined && nextId === previousId,
265 };
266 }
267 return undefined;
268 }),
269
270 read: Effect.fn(function* ({ id, olds, output }) {
271 const env = yield* GcpEnvironment.current;
272 const location = normalizeLocation(olds?.location ?? output?.location, env.region);
273 const batchId = yield* toPhysicalId(
274 id,
275 olds?.batchId,
276 output?.batchId,
277 MAX_WORKLOAD_ID_LENGTH,
278 "batch",
279 );
280 const name = output?.name ?? resourceName(env.project, location, batchId);
281 const existing = yield* getByName(name);
282 if (existing === undefined) return undefined;
283 const attrs = toAttrs(existing, env.project, location);
284 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
285 }),
286
287 list: () =>
288 Effect.gen(function* () {
289 const env = yield* GcpEnvironment.current;
290 const pages = yield* Effect.forEach(
291 [...new Set([env.region, ...LIST_LOCATIONS])],
292 (location) => listLocation(env.project, location),
293 { concurrency: 4 },
294 );
295 return pages.flat();
296 }),
297
298 reconcile: Effect.fn(function* ({ id, news, output }) {
299 const env = yield* GcpEnvironment.current;
300 const location = normalizeLocation(news.location ?? output?.location, env.region);
301 const batchId = yield* toPhysicalId(
302 id,
303 news.batchId,
304 output?.batchId,
305 MAX_WORKLOAD_ID_LENGTH,
306 "batch",
307 );
308 const name = resourceName(env.project, location, batchId);
309 const desiredLabels = {
310 ...toLabels(news.labels),
311 ...(yield* createInternalLabels(id)),
312 };
313 const desired = desiredBody(news, desiredLabels);
314
315 let current = yield* getByName(name);
316
317 if (current === undefined) {
318 const created = yield* dataproc
319 .createProjectsLocationsBatches({
320 parent: locationParent(env.project, location),
321 batchId,
322 body: desired,
323 })
324 .pipe(Effect.catchTag("Conflict", () => Effect.undefined));
325 if (created) {
326 yield* waitForOperation(created);
327 }
328 current = yield* waitUntilExists(getByName(name), name);
329 }
330
331 if (current === undefined) {
332 return yield* new BatchNotResolved({ resourceName: name });
333 }
334
335 return toAttrs(current, env.project, location);
336 }),
337
338 delete: Effect.fn(function* ({ output }) {
339 if (!output.name) return;
340 yield* waitUntilTerminal(output.name);
341 yield* dataproc.deleteProjectsLocationsBatches({ name: output.name }).pipe(
342 Effect.retry({
343 while: (error) => error._tag === "Conflict" || error._tag === "BadRequest",
344 times: 8,
345 schedule: Schedule.spaced("5 seconds"),
346 }),
347 Effect.catchTag("NotFound", () => Effect.void),
348 );
349 yield* waitUntilGone(getByName(output.name), output.name);
350 }),
351 });