@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 362 lines 11.9 KB
Raw
1 import * as aiplatform from "@distilled.cloud/gcp/aiplatform_v1";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import { tagRecord } from "alchemy/Tags";
7 import * as Data from "effect/Data";
8 import * as Effect from "effect/Effect";
9 import * as Schedule from "effect/Schedule";
10 import { GcpEnvironment } from "../Environment.ts";
11 import { createInternalLabels, hasAlchemyLabels, toLabels } from "../Labels.ts";
12 import type { Providers } from "../Providers.ts";
13 import { listLocations } from "./names.ts";
14 import { waitForOperation } from "./operations.ts";
15 import {
16 AiPlatformStillExists,
17 MAX_PIPELINE_JOB_ID_LENGTH,
18 collectPages,
19 jsonEqual,
20 locationParent,
21 normalizeLocation,
22 parseResourceName,
23 toPhysicalId,
24 userLabels,
25 type EncryptionSpec,
26 } from "./shared.ts";
27
28 const COLLECTION = "pipelineJobs";
29
30 export type PipelineRuntimeConfig = {
31 /** Cloud Storage root for pipeline artifacts. */
32 gcsOutputDirectory?: string;
33 /** Runtime parameters (schema 2.1.0+). */
34 parameterValues?: Record<string, unknown>;
35 /** Failure policy (`PIPELINE_FAILURE_POLICY_FAIL_SLOW` / `_FAIL_FAST`). */
36 failurePolicy?: string;
37 /** Input artifacts keyed by name. */
38 inputArtifacts?: Record<string, { artifactId?: string }>;
39 };
40
41 export type PipelineJobProps = {
42 /**
43 * Job id. If omitted, a unique RFC1035 name is generated. Less than
44 * 128 characters, matching `/[a-z0-9-]+/`. Immutable — changing it
45 * replaces the job (a new run is started).
46 */
47 pipelineJobId?: string;
48 /**
49 * Vertex AI location. Immutable — changing it replaces the job.
50 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
51 */
52 location?: string;
53 /**
54 * Display name (max 128 UTF-8 characters). Defaults to the job id.
55 */
56 displayName?: string;
57 /**
58 * User labels. Alchemy ownership labels are merged in automatically.
59 */
60 labels?: Record<string, string>;
61 /**
62 * Compiled Kubeflow pipeline spec as a JSON object.
63 */
64 pipelineSpec?: Record<string, unknown>;
65 /**
66 * Template URI (Vertex Template Registry or Gallery). Used when
67 * `pipelineSpec` is omitted.
68 */
69 templateUri?: string;
70 /**
71 * Runtime config (output directory, parameters, failure policy).
72 */
73 runtimeConfig?: PipelineRuntimeConfig;
74 /**
75 * Service account the pipeline runs as.
76 */
77 serviceAccount?: string;
78 /**
79 * VPC network to peer.
80 */
81 network?: string;
82 /**
83 * Reserved IP ranges under the VPC.
84 */
85 reservedIpRanges?: string[];
86 /**
87 * Customer-managed encryption key.
88 */
89 encryptionSpec?: EncryptionSpec;
90 /**
91 * Run component-level validations before create.
92 */
93 preflightValidations?: boolean;
94 };
95
96 export type PipelineJob = Resource<
97 "GCP.AIPlatform.PipelineJob",
98 PipelineJobProps,
99 {
100 /** Full resource name. */
101 name: string;
102 /** Job id (last path segment). */
103 pipelineJobId: string;
104 /** Project id. */
105 project: string;
106 /** Location id. */
107 location: string;
108 /** Display name. */
109 displayName: string | undefined;
110 /** User labels (Alchemy ownership labels stripped). */
111 labels: Record<string, string>;
112 /** Pipeline state (`PIPELINE_STATE_RUNNING`, …). */
113 state: string | undefined;
114 /** Template URI. */
115 templateUri: string | undefined;
116 /** Service account. */
117 serviceAccount: string | undefined;
118 /** RFC3339 creation timestamp. */
119 createTime: string | undefined;
120 /** RFC3339 start timestamp. */
121 startTime: string | undefined;
122 /** RFC3339 end timestamp. */
123 endTime: string | undefined;
124 /** RFC3339 last-update timestamp. */
125 updateTime: string | undefined;
126 },
127 never,
128 Providers
129 >;
130
131 /**
132 * A Vertex AI PipelineJob — a compiled Kubeflow pipeline that runs
133 * immediately on create.
134 *
135 * There is no update API. Changing identity (`pipelineJobId`, `location`)
136 * or the pipeline spec / template URI replaces the job.
137 *
138 * ### Creating a Pipeline Job
139 * **Example:** Run a compiled spec
140 * ```typescript
141 * const job = yield* GCP.AIPlatform.PipelineJob("Train", {
142 * pipelineSpec: compiled,
143 * runtimeConfig: { gcsOutputDirectory: "gs://bucket/pipeline-out" },
144 * labels: { env: "prod" },
145 * });
146 * ```
147 *
148 * @resource
149 * @category AIPlatform
150 */
151 export const PipelineJob = Resource<PipelineJob>("GCP.AIPlatform.PipelineJob");
152
153 export class PipelineJobNotResolved extends Data.TaggedError(
154 "GCP.AIPlatform.PipelineJobNotResolved",
155 )<{
156 resourceName: string;
157 }> {}
158
159 const resourceName = (project: string, location: string, id: string) =>
160 `${locationParent(project, location)}/${COLLECTION}/${id}`;
161
162 const toAttrs = (job: aiplatform.GoogleCloudAiplatformV1PipelineJob, project: string) => {
163 const name = job.name ?? "";
164 const parsed = parseResourceName(name, COLLECTION);
165 return {
166 name,
167 pipelineJobId: parsed.id,
168 project: parsed.project || project,
169 location: parsed.location,
170 displayName: job.displayName,
171 labels: userLabels(job.labels),
172 state: job.state,
173 templateUri: job.templateUri,
174 serviceAccount: job.serviceAccount,
175 createTime: job.createTime,
176 startTime: job.startTime,
177 endTime: job.endTime,
178 updateTime: job.updateTime,
179 };
180 };
181
182 const getByName = (name: string) =>
183 aiplatform
184 .getProjectsLocationsPipelineJobs({ name })
185 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
186
187 const TERMINAL_STATES = new Set([
188 "PIPELINE_STATE_SUCCEEDED",
189 "PIPELINE_STATE_FAILED",
190 "PIPELINE_STATE_CANCELLED",
191 ]);
192
193 class PipelineJobNotTerminal extends Data.TaggedError("GCP.AIPlatform.PipelineJobNotTerminal")<{
194 resourceName: string;
195 state: string | undefined;
196 }> {}
197
198 /**
199 * A pending or running job cannot be deleted: cancel it and wait until it
200 * settles (cancellation takes up to a few minutes).
201 */
202 const cancelUntilTerminal = (name: string) =>
203 Effect.gen(function* () {
204 const job = yield* getByName(name);
205 if (job === undefined || TERMINAL_STATES.has(job.state ?? "")) return;
206 yield* aiplatform.cancelProjectsLocationsPipelineJobs({ name, body: {} }).pipe(
207 Effect.catchTags({
208 NotFound: () => Effect.void,
209 // The job can reach a terminal state between the read above and the
210 // cancel ("… is in state JOB_STATE_FAILED and cannot be canceled");
211 // the wait below re-reads the state and still fails if it is not
212 // terminal.
213 BadRequest: () => Effect.void,
214 }),
215 );
216 yield* getByName(name).pipe(
217 Effect.flatMap((current) =>
218 current === undefined || TERMINAL_STATES.has(current.state ?? "")
219 ? Effect.void
220 : Effect.fail(new PipelineJobNotTerminal({ resourceName: name, state: current.state })),
221 ),
222 Effect.retry({
223 while: (error) => error._tag === "GCP.AIPlatform.PipelineJobNotTerminal",
224 times: 36,
225 schedule: Schedule.spaced("5 seconds"),
226 }),
227 );
228 });
229
230 const waitUntilGone = (name: string) =>
231 getByName(name).pipe(
232 Effect.filterOrFail(
233 (job) => job === undefined,
234 () => new AiPlatformStillExists({ resourceName: name }),
235 ),
236 Effect.asVoid,
237 Effect.retry({
238 while: (error) => error._tag === "GCP.AIPlatform.StillExists",
239 times: 10,
240 schedule: Schedule.spaced("2 seconds"),
241 }),
242 );
243
244 export const PipelineJobProvider = () =>
245 Provider.succeed(PipelineJob, {
246 stables: ["name", "pipelineJobId", "project", "location", "createTime"],
247
248 diff: Effect.fn(function* ({ news, olds, output }) {
249 const env = yield* GcpEnvironment.current;
250 if (!isResolved(news)) return undefined;
251 const previousId = olds?.pipelineJobId ?? output?.pipelineJobId;
252 const nextId = news.pipelineJobId ?? previousId;
253 const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region);
254 const nextLocation = normalizeLocation(news.location ?? output?.location, env.region);
255 const specChanged =
256 olds !== undefined &&
257 (!jsonEqual(news.pipelineSpec, olds.pipelineSpec) ||
258 (news.templateUri ?? "") !== (olds.templateUri ?? ""));
259 const replace =
260 (previousId !== undefined && nextId !== undefined && nextId !== previousId) ||
261 previousLocation !== nextLocation ||
262 specChanged;
263 if (!replace) return undefined;
264 return {
265 action: "replace" as const,
266 deleteFirst:
267 previousLocation === nextLocation && previousId !== undefined && nextId === previousId,
268 };
269 }),
270
271 read: Effect.fn(function* ({ id, olds, output }) {
272 const env = yield* GcpEnvironment.current;
273 const jobId = yield* toPhysicalId(
274 id,
275 olds?.pipelineJobId,
276 output?.pipelineJobId,
277 MAX_PIPELINE_JOB_ID_LENGTH,
278 );
279 const location = normalizeLocation(olds?.location ?? output?.location, env.region);
280 const name = output?.name ?? resourceName(env.project, location, jobId);
281 const existing = yield* getByName(name);
282 if (existing === undefined) return undefined;
283 const attrs = toAttrs(existing, env.project);
284 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
285 }),
286
287 list: () =>
288 Effect.gen(function* () {
289 const env = yield* GcpEnvironment.current;
290 const pages = (yield* Effect.forEach(listLocations(env.region), (location) =>
291 collectPages(
292 aiplatform.listProjectsLocationsPipelineJobs.pages({
293 parent: locationParent(env.project, location),
294 pageSize: 100,
295 }),
296 ).pipe(Effect.catchTag("NotFound", () => Effect.succeed([]))),
297 )).flat();
298 return pages.flatMap((page) =>
299 (page.pipelineJobs ?? [])
300 .filter((job) =>
301 Object.keys(job.labels ?? {}).some((key) => key.startsWith("alchemy-")),
302 )
303 .map((job) => toAttrs(job, env.project)),
304 );
305 }),
306
307 reconcile: Effect.fn(function* ({ id, news, output }) {
308 const env = yield* GcpEnvironment.current;
309 const jobId = yield* toPhysicalId(
310 id,
311 news.pipelineJobId,
312 output?.pipelineJobId,
313 MAX_PIPELINE_JOB_ID_LENGTH,
314 );
315 const location = normalizeLocation(news.location ?? output?.location, env.region);
316 const name = resourceName(env.project, location, jobId);
317 const desiredLabels = {
318 ...toLabels(news.labels),
319 ...(yield* createInternalLabels(id)),
320 };
321
322 let current = yield* getByName(output?.name ?? name);
323
324 if (current === undefined) {
325 const created = yield* aiplatform
326 .createProjectsLocationsPipelineJobs({
327 parent: locationParent(env.project, location),
328 pipelineJobId: jobId,
329 body: {
330 displayName: news.displayName ?? jobId,
331 labels: desiredLabels,
332 pipelineSpec: news.pipelineSpec,
333 templateUri: news.templateUri,
334 runtimeConfig: news.runtimeConfig,
335 serviceAccount: news.serviceAccount,
336 network: news.network,
337 reservedIpRanges: news.reservedIpRanges,
338 encryptionSpec: news.encryptionSpec,
339 preflightValidations: news.preflightValidations,
340 },
341 })
342 .pipe(Effect.catchTag("Conflict", () => getByName(name)));
343 current = created ?? undefined;
344 }
345
346 if (current === undefined) {
347 return yield* new PipelineJobNotResolved({ resourceName: name });
348 }
349 return toAttrs(current, env.project);
350 }),
351
352 delete: Effect.fn(function* ({ output }) {
353 yield* cancelUntilTerminal(output.name);
354 const operation = yield* aiplatform
355 .deleteProjectsLocationsPipelineJobs({ name: output.name })
356 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
357 if (operation !== undefined) {
358 yield* waitForOperation(operation, { notFoundOk: true });
359 }
360 yield* waitUntilGone(output.name);
361 }),
362 });