The best way to control Claude Code, Codex and more from any computer or phone

main
ts 423 lines 14.2 KB
Raw
1 import * as datapipelines from "@distilled.cloud/gcp/datapipelines_v1";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import * as Data from "effect/Data";
7 import * as Effect from "effect/Effect";
8 import { GcpEnvironment } from "../Environment.ts";
9 import { createInternalLabels } from "../Labels.ts";
10 import type { Providers } from "../Providers.ts";
11 import {
12 DEFAULT_STATE,
13 DEFAULT_TYPE,
14 findOwnedPipeline,
15 getPipeline,
16 ignoreMissing,
17 jsonEqual,
18 listOwnedPipelines,
19 locationParent,
20 normalizeLocation,
21 ownedByAlchemy,
22 ownershipSources,
23 parseName,
24 replaceOnIdentity,
25 resourceName,
26 sameText,
27 scheduleKey,
28 toDisplayName,
29 toPipelineId,
30 updateMaskOf,
31 userSources,
32 } from "./internal.ts";
33
34 export type PipelineType = datapipelines.GoogleCloudDatapipelinesV1PipelineTypeEnum | (string & {});
35
36 export type PipelineState =
37 | datapipelines.GoogleCloudDatapipelinesV1PipelineStateEnum
38 | (string & {});
39
40 export type ScheduleSpec = {
41 /**
42 * Unix-cron schedule (for example `"0 0 * * *"`). When set, Data
43 * Pipelines creates an internal Cloud Scheduler job.
44 */
45 schedule?: string;
46 /**
47 * tz-database time zone used to interpret `schedule`. Empty uses UTC.
48 */
49 timeZone?: string;
50 };
51
52 export type Workload = datapipelines.GoogleCloudDatapipelinesV1Workload;
53
54 export type PipelineProps = {
55 /**
56 * Pipeline id (the `{pipeline}` segment of
57 * `projects/{project}/locations/{location}/pipelines/{pipeline}`). If
58 * omitted, a unique RFC1035 name is generated from the stack, stage,
59 * and logical id. Immutable — changing it replaces the pipeline.
60 */
61 pipelineId?: string;
62 /**
63 * Location (`us-central1`, …). Immutable — changing it replaces the
64 * pipeline. `US-CENTRAL1` is accepted and normalized to `us-central1`.
65 * Data Pipelines is only available in App Engine regions.
66 * @default the stack's GCP region (`GCP.Region`, profile region, `us-central1`)
67 */
68 location?: string;
69 /**
70 * Display name. Letters, numbers, hyphens, and underscores only. If
71 * omitted, Alchemy uses the generated pipeline id.
72 */
73 displayName?: string;
74 /**
75 * Pipeline type (`PIPELINE_TYPE_BATCH` or `PIPELINE_TYPE_STREAMING`).
76 * Batch pipelines run on `scheduleInfo`; streaming pipelines launch a
77 * job at create time. Immutable — changing it replaces the pipeline.
78 * @default "PIPELINE_TYPE_BATCH"
79 */
80 type?: PipelineType;
81 /**
82 * Desired execution state (`STATE_ACTIVE`, `STATE_PAUSED`,
83 * `STATE_ARCHIVED`, …). Applied on create. Later archive requests call
84 * `pipelines.stop` (permanent). State cannot be patched.
85 * @default "STATE_ACTIVE"
86 */
87 state?: PipelineState;
88 /**
89 * Cron used to launch batch jobs. Omit to manage jobs with
90 * `pipelines.run` instead of Cloud Scheduler.
91 */
92 scheduleInfo?: ScheduleSpec;
93 /**
94 * Service account email for the internal Cloud Scheduler job. Defaults
95 * to the Compute Engine default service account.
96 */
97 schedulerServiceAccountEmail?: string;
98 /**
99 * Dataflow classic or Flex template used to launch jobs. Exactly one of
100 * `dataflowLaunchTemplateRequest` or `dataflowFlexTemplateRequest`.
101 */
102 workload?: Workload;
103 /**
104 * Source annotations (for example Dataplex). Immutable. Alchemy
105 * ownership (`alchemy-stack` / `alchemy-stage` / `alchemy-id`) is
106 * merged in automatically because pipelines have no labels field.
107 */
108 pipelineSources?: Record<string, string>;
109 };
110
111 export type Pipeline = Resource<
112 "GCP.DataPipelines.Pipeline",
113 PipelineProps,
114 {
115 /** Full resource name `projects/{project}/locations/{location}/pipelines/{pipeline}`. */
116 name: string;
117 /** Pipeline id (last path segment). */
118 pipelineId: string;
119 /** Location id. */
120 location: string;
121 /** Project id. */
122 project: string;
123 /** Parent `projects/{project}/locations/{location}`. */
124 parent: string;
125 /** Display name. */
126 displayName: string | undefined;
127 /** Pipeline type. */
128 type: string | undefined;
129 /** Execution state. */
130 state: string | undefined;
131 /** Cron schedule, if configured. */
132 scheduleInfo: ScheduleSpec | undefined;
133 /** Scheduler service account email, if set. */
134 schedulerServiceAccountEmail: string | undefined;
135 /** Dataflow template used to launch jobs. */
136 workload: Workload | undefined;
137 /** User source annotations (Alchemy ownership keys stripped). */
138 pipelineSources: Record<string, string>;
139 /** Number of launched jobs. */
140 jobCount: number | undefined;
141 /** RFC3339 creation timestamp. */
142 createTime: string | undefined;
143 /** RFC3339 last-update timestamp. */
144 lastUpdateTime: string | undefined;
145 },
146 never,
147 Providers
148 >;
149
150 /**
151 * A Data Pipelines pipeline that launches recurring Dataflow jobs.
152 *
153 * Pipelines have no labels field, so Alchemy stamps ownership into
154 * `pipelineSources` for `list` / nuke. Pipeline id, location, and type
155 * are identity — changing any of them replaces the pipeline. Display
156 * name, workload, schedule, and scheduler service account update in
157 * place. Archiving uses `pipelines.stop` and is permanent. Creating a
158 * pipeline requires the Data Pipelines API
159 * (`datapipelines.googleapis.com`).
160 *
161 * ### Creating a Pipeline
162 * **Example:** Generated name
163 * ```typescript
164 * const pipeline = yield* GCP.DataPipelines.Pipeline("Batch", {
165 * type: "PIPELINE_TYPE_BATCH",
166 * });
167 * ```
168 *
169 * **Example:** Classic Word Count template on a yearly cron
170 * ```typescript
171 * const pipeline = yield* GCP.DataPipelines.Pipeline("WordCount", {
172 * pipelineId: "word-count",
173 * type: "PIPELINE_TYPE_BATCH",
174 * displayName: "word-count",
175 * scheduleInfo: { schedule: "0 0 1 1 *", timeZone: "UTC" },
176 * workload: {
177 * dataflowLaunchTemplateRequest: {
178 * projectId: "my-project",
179 * location: "us-central1",
180 * gcsPath: "gs://dataflow-templates/latest/Word_Count",
181 * launchParameters: {
182 * jobName: "word-count",
183 * parameters: {
184 * inputFile: "gs://dataflow-samples/shakespeare/kinglear.txt",
185 * output: "gs://my-bucket/out",
186 * },
187 * environment: { tempLocation: "gs://my-bucket/tmp" },
188 * },
189 * },
190 * },
191 * });
192 * ```
193 *
194 * ### Updating a Pipeline
195 * Change props on the same logical id; the engine keeps the physical id.
196 *
197 * **Example:** Rename and change the schedule
198 * ```typescript
199 * const pipeline = yield* GCP.DataPipelines.Pipeline("WordCount", {
200 * type: "PIPELINE_TYPE_BATCH",
201 * displayName: "word-count-v2",
202 * scheduleInfo: { schedule: "0 0 1 2 *", timeZone: "UTC" },
203 * });
204 * ```
205 *
206 * @resource
207 * @category DataPipelines
208 */
209 export const Pipeline = Resource<Pipeline>("GCP.DataPipelines.Pipeline");
210
211 export class PipelineNotResolved extends Data.TaggedError("GCP.DataPipelines.PipelineNotResolved")<{
212 resourceName: string;
213 }> {}
214
215 const toAttrs = (
216 pipeline: datapipelines.GoogleCloudDatapipelinesV1Pipeline,
217 project: string,
218 region: string,
219 ) => {
220 const name = pipeline.name ?? "";
221 const parsed = parseName(name, region);
222 return {
223 name,
224 pipelineId: parsed.pipelineId,
225 location: parsed.location,
226 project: parsed.project || project,
227 parent: parsed.parent || locationParent(project, parsed.location),
228 displayName: pipeline.displayName,
229 type: pipeline.type,
230 state: pipeline.state,
231 scheduleInfo: pipeline.scheduleInfo
232 ? {
233 schedule: pipeline.scheduleInfo.schedule,
234 timeZone: pipeline.scheduleInfo.timeZone,
235 }
236 : undefined,
237 schedulerServiceAccountEmail: pipeline.schedulerServiceAccountEmail,
238 workload: pipeline.workload,
239 pipelineSources: userSources(pipeline.pipelineSources),
240 jobCount: pipeline.jobCount,
241 createTime: pipeline.createTime,
242 lastUpdateTime: pipeline.lastUpdateTime,
243 };
244 };
245
246 const fillWorkload = (
247 workload: Workload | undefined,
248 project: string,
249 location: string,
250 ): Workload | undefined => {
251 if (workload === undefined) return undefined;
252 const classic = workload.dataflowLaunchTemplateRequest;
253 const flex = workload.dataflowFlexTemplateRequest;
254 return {
255 dataflowLaunchTemplateRequest: classic
256 ? {
257 ...classic,
258 projectId: classic.projectId ?? project,
259 location: classic.location ?? location,
260 }
261 : undefined,
262 dataflowFlexTemplateRequest: flex
263 ? {
264 ...flex,
265 projectId: flex.projectId ?? project,
266 location: flex.location ?? location,
267 }
268 : undefined,
269 };
270 };
271
272 const desiredSchedule = (
273 info: ScheduleSpec | undefined,
274 ): datapipelines.GoogleCloudDatapipelinesV1ScheduleSpec | undefined => {
275 if (info === undefined) return undefined;
276 if ((info.schedule ?? "") === "" && (info.timeZone ?? "") === "") {
277 return undefined;
278 }
279 return {
280 schedule: info.schedule,
281 timeZone: info.timeZone,
282 };
283 };
284
285 export const PipelineProvider = () =>
286 Provider.succeed(Pipeline, {
287 stables: ["name", "pipelineId", "location", "project", "parent", "createTime"],
288
289 diff: Effect.fn(function* ({ news, olds, output }) {
290 if (!isResolved(news)) return undefined;
291 const env = yield* GcpEnvironment.current;
292 const previousLocation = normalizeLocation(olds?.location ?? output?.location, env.region);
293 const nextLocation = normalizeLocation(
294 news.location ?? olds?.location ?? output?.location,
295 env.region,
296 );
297 const previousType = olds?.type ?? output?.type;
298 const nextType = news.type ?? previousType;
299 return replaceOnIdentity({
300 previousId: olds?.pipelineId ?? output?.pipelineId,
301 nextId: news.pipelineId,
302 previousLocation,
303 nextLocation,
304 previousType,
305 nextType,
306 });
307 }),
308
309 read: Effect.fn(function* ({ id, olds, output }) {
310 const env = yield* GcpEnvironment.current;
311 const pipelineId = yield* toPipelineId(id, olds?.pipelineId, output?.pipelineId);
312 const location = normalizeLocation(olds?.location ?? output?.location, env.region);
313 const name = output?.name ?? resourceName(env.project, location, pipelineId);
314 const existing = yield* findOwnedPipeline(id, env.project, env.region, name);
315 if (existing === undefined) return undefined;
316 const attrs = toAttrs(existing, env.project, env.region);
317 return (yield* ownedByAlchemy(id, existing.pipelineSources)) ? attrs : Unowned(attrs);
318 }),
319
320 list: () =>
321 Effect.gen(function* () {
322 const env = yield* GcpEnvironment.current;
323 const rows = yield* listOwnedPipelines(env.project, env.region);
324 return rows.map((row) => toAttrs(row, env.project, env.region));
325 }),
326
327 reconcile: Effect.fn(function* ({ id, news, output }) {
328 const env = yield* GcpEnvironment.current;
329 const pipelineId = yield* toPipelineId(id, news.pipelineId, output?.pipelineId);
330 const location = normalizeLocation(news.location ?? output?.location, env.region);
331 const parent = locationParent(env.project, location);
332 const name = resourceName(env.project, location, pipelineId);
333 const displayName = toDisplayName(news.displayName, pipelineId);
334 const type = news.type ?? output?.type ?? DEFAULT_TYPE;
335 const state = news.state ?? output?.state ?? DEFAULT_STATE;
336 const ownership = yield* createInternalLabels(id);
337 const pipelineSources = ownershipSources(
338 ownership,
339 news.pipelineSources ?? output?.pipelineSources,
340 );
341 const scheduleInfo = desiredSchedule(news.scheduleInfo);
342 const workload = fillWorkload(news.workload, env.project, location);
343 const schedulerServiceAccountEmail = news.schedulerServiceAccountEmail;
344
345 let current = yield* findOwnedPipeline(id, env.project, env.region, output?.name ?? name);
346
347 if (current === undefined) {
348 const created = yield* datapipelines
349 .createProjectsLocationsPipelines({
350 parent,
351 body: {
352 name,
353 displayName,
354 type,
355 state,
356 scheduleInfo,
357 schedulerServiceAccountEmail,
358 workload,
359 pipelineSources,
360 },
361 })
362 .pipe(Effect.catchTag("Conflict", () => getPipeline(name)));
363 current = created ?? undefined;
364 }
365
366 if (current === undefined) {
367 return yield* new PipelineNotResolved({ resourceName: name });
368 }
369
370 const currentName = current.name ?? name;
371 const displayChanged = !sameText(current.displayName, displayName);
372 const scheduleChanged =
373 news.scheduleInfo !== undefined &&
374 !jsonEqual(scheduleKey(current.scheduleInfo), scheduleKey(scheduleInfo));
375 const schedulerChanged =
376 news.schedulerServiceAccountEmail !== undefined &&
377 !sameText(current.schedulerServiceAccountEmail, schedulerServiceAccountEmail);
378 const workloadChanged = news.workload !== undefined && !jsonEqual(current.workload, workload);
379 const updateMask = updateMaskOf(
380 displayChanged ? "displayName" : undefined,
381 scheduleChanged ? "scheduleInfo" : undefined,
382 schedulerChanged ? "schedulerServiceAccountEmail" : undefined,
383 workloadChanged ? "workload" : undefined,
384 );
385
386 if (updateMask.length > 0) {
387 current = yield* datapipelines.patchProjectsLocationsPipelines({
388 name: currentName,
389 updateMask,
390 body: {
391 displayName,
392 scheduleInfo,
393 schedulerServiceAccountEmail,
394 workload,
395 },
396 });
397 }
398
399 const wantsArchived = state === "STATE_ARCHIVED";
400 if (wantsArchived && current.state !== "STATE_ARCHIVED") {
401 current = yield* datapipelines
402 .stopProjectsLocationsPipelines({
403 name: currentName,
404 body: {},
405 })
406 .pipe(
407 Effect.catchTag("NotFound", () => getPipeline(currentName)),
408 Effect.flatMap((row) =>
409 row !== undefined
410 ? Effect.succeed(row)
411 : new PipelineNotResolved({ resourceName: currentName }),
412 ),
413 );
414 }
415
416 return toAttrs(current, env.project, env.region);
417 }),
418
419 delete: Effect.fn(function* ({ output }) {
420 if (!output.name) return;
421 yield* ignoreMissing(datapipelines.deleteProjectsLocationsPipelines({ name: output.name }));
422 }),
423 });