The best way to control Claude Code, Codex and more from any computer or phone

main
ts 491 lines 17.9 KB
Raw
1 import * as ds from "@distilled.cloud/gcp/datastream_v1";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import { tagRecord } from "alchemy/Tags";
7 import * as Effect from "effect/Effect";
8 import * as Schedule from "effect/Schedule";
9 import { GcpEnvironment } from "../Environment.ts";
10 import { createInternalLabels, diffLabels, hasAlchemyLabels, toLabels } from "../Labels.ts";
11 import type { Providers } from "../Providers.ts";
12 import {
13 collectPages,
14 connectionProfileOf,
15 emptyMessage,
16 fieldMask,
17 fingerprint,
18 hasAlchemyLabelMap,
19 lastSegment,
20 listAtLocation,
21 locationParent,
22 normalizeLocation,
23 parseName,
24 replaceOnIdentity,
25 ResourceNotResolved,
26 settleOperation,
27 toPhysicalId,
28 userLabels,
29 waitUntilExists,
30 waitUntilGone,
31 } from "./internal.ts";
32
33 export type StreamState = ds.StreamStateEnum | (string & {});
34 export type SourceConfig = ds.SourceConfig;
35 export type DestinationConfig = ds.DestinationConfig;
36 export type BackfillAllStrategy = ds.BackfillAllStrategy;
37 export type BigQueryDestinationConfig = ds.BigQueryDestinationConfig;
38 export type GcsDestinationConfig = ds.GcsDestinationConfig;
39 export type MysqlSourceConfig = ds.MysqlSourceConfig;
40 export type PostgresqlSourceConfig = ds.PostgresqlSourceConfig;
41 export type OracleSourceConfig = ds.OracleSourceConfig;
42 export type SqlServerSourceConfig = ds.SqlServerSourceConfig;
43 export type MongodbSourceConfig = ds.MongodbSourceConfig;
44 export type SpannerSourceConfig = ds.SpannerSourceConfig;
45 export type SalesforceSourceConfig = ds.SalesforceSourceConfig;
46 export type SalesforceMarketingCloudSourceConfig = ds.SalesforceMarketingCloudSourceConfig;
47 export type ServiceNowSourceConfig = ds.ServiceNowSourceConfig;
48 export type RuleSet = ds.RuleSet;
49 export type DatastreamError = ds.Datastream_Error;
50
51 export type StreamProps = {
52 /**
53 * Stream id (the `{stream}` segment of
54 * `projects/{project}/locations/{location}/streams/{stream}`). If
55 * omitted, a unique RFC1035 name is generated. Immutable — changing it
56 * replaces the stream.
57 */
58 streamId?: string;
59 /**
60 * Region (`us-central1`, …). Immutable — changing it replaces the
61 * stream. `US-CENTRAL1` is accepted and normalized to `us-central1`.
62 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
63 */
64 location?: string;
65 /**
66 * User-friendly display name.
67 */
68 displayName?: string;
69 /**
70 * User labels. Alchemy ownership labels are merged in automatically.
71 */
72 labels?: Record<string, string>;
73 /**
74 * Source connection profile and engine-specific include/exclude
75 * objects. `sourceConnectionProfile` may be a full name or id
76 * (combined with `location`). Changing the source profile replaces the
77 * stream; other source config updates in place.
78 */
79 sourceConfig: SourceConfig;
80 /**
81 * Destination connection profile and BigQuery / GCS writer config.
82 * `destinationConnectionProfile` may be a full name or id (combined
83 * with `location`). Changing the destination profile replaces the
84 * stream; other destination config updates in place.
85 */
86 destinationConfig: DestinationConfig;
87 /**
88 * Automatically backfill objects included in the source config.
89 * Mutually exclusive with `backfillNone`.
90 */
91 backfillAll?: BackfillAllStrategy;
92 /**
93 * Do not automatically backfill any objects. Mutually exclusive with
94 * `backfillAll`.
95 */
96 backfillNone?: Record<string, never>;
97 /**
98 * Customer-managed Cloud KMS key used to encrypt stream data.
99 * Immutable — changing it replaces the stream.
100 */
101 customerManagedEncryptionKey?: string;
102 /**
103 * Customization rule sets applied to stream objects.
104 */
105 ruleSets?: ReadonlyArray<RuleSet>;
106 /**
107 * Desired stream state (`NOT_STARTED`, `RUNNING`, `PAUSED`, …).
108 */
109 state?: StreamState;
110 /**
111 * Create or update without validating connectivity.
112 */
113 force?: boolean;
114 /**
115 * Validate without creating or updating resources.
116 */
117 validateOnly?: boolean;
118 };
119
120 export type Stream = Resource<
121 "GCP.Datastream.Stream",
122 StreamProps,
123 {
124 /** Full resource name. */
125 name: string;
126 /** Stream id (last path segment). */
127 streamId: string;
128 /** Project id. */
129 project: string;
130 /** Location id. */
131 location: string;
132 /** User-friendly display name. */
133 displayName: string | undefined;
134 /** User labels (Alchemy ownership labels stripped). */
135 labels: Record<string, string>;
136 /** Source connection profile and engine config. */
137 sourceConfig: SourceConfig | undefined;
138 /** Destination connection profile and writer config. */
139 destinationConfig: DestinationConfig | undefined;
140 /** Backfill-all strategy. */
141 backfillAll: BackfillAllStrategy | undefined;
142 /** Backfill-none strategy. */
143 backfillNone: Record<string, never> | undefined;
144 /** Customer-managed encryption key. */
145 customerManagedEncryptionKey: string | undefined;
146 /** Customization rule sets. */
147 ruleSets: ReadonlyArray<RuleSet> | undefined;
148 /** Server-reported state. */
149 state: string | undefined;
150 /** Errors on the stream. */
151 errors: ReadonlyArray<DatastreamError> | undefined;
152 /** RFC3339 last recovery timestamp. */
153 lastRecoveryTime: string | undefined;
154 /** Whether the stream satisfies physical zone isolation. */
155 satisfiesPzi: boolean | undefined;
156 /** Whether the stream satisfies physical zone separation. */
157 satisfiesPzs: boolean | undefined;
158 /** RFC3339 creation timestamp. */
159 createTime: string | undefined;
160 /** RFC3339 last-update timestamp. */
161 updateTime: string | undefined;
162 },
163 never,
164 Providers
165 >;
166
167 /**
168 * A Datastream stream: continuously replicates data from a source
169 * connection profile to a BigQuery or Cloud Storage destination.
170 *
171 * `streamId`, `location`, `customerManagedEncryptionKey`, and the source
172 * / destination connection profile identities are replacement triggers.
173 * Display name, labels, include/exclude objects, backfill strategy,
174 * rule sets, and desired `state` update in place.
175 *
176 * ### Creating a Stream
177 * **Example:** MySQL to BigQuery
178 * ```typescript
179 * const source = yield* GCP.Datastream.ConnectionProfile("MysqlSrc", {
180 * mysqlProfile: {
181 * hostname: "10.0.0.8",
182 * username: "datastream",
183 * password: process.env.MYSQL_PASSWORD,
184 * },
185 * force: true,
186 * });
187 * const dest = yield* GCP.Datastream.ConnectionProfile("BqDest", {
188 * bigqueryProfile: {},
189 * });
190 * const stream = yield* GCP.Datastream.Stream("MysqlToBq", {
191 * sourceConfig: {
192 * sourceConnectionProfile: source.name,
193 * mysqlSourceConfig: {
194 * includeObjects: { mysqlDatabases: [{ database: "app" }] },
195 * },
196 * },
197 * destinationConfig: {
198 * destinationConnectionProfile: dest.name,
199 * bigqueryDestinationConfig: {
200 * singleTargetDataset: { datasetId: "app_replica" },
201 * },
202 * },
203 * backfillAll: {},
204 * force: true,
205 * });
206 * ```
207 *
208 * @resource
209 * @category Datastream
210 */
211 export const Stream = Resource<Stream>("GCP.Datastream.Stream");
212
213 const resourceName = (project: string, location: string, streamId: string) =>
214 `${locationParent(project, location)}/streams/${streamId}`;
215
216 const expandSource = (config: SourceConfig, project: string, location: string): SourceConfig => ({
217 ...config,
218 sourceConnectionProfile: config.sourceConnectionProfile
219 ? connectionProfileOf(config.sourceConnectionProfile, project, location)
220 : undefined,
221 });
222
223 const expandDestination = (
224 config: DestinationConfig,
225 project: string,
226 location: string,
227 ): DestinationConfig => ({
228 ...config,
229 destinationConnectionProfile: config.destinationConnectionProfile
230 ? connectionProfileOf(config.destinationConnectionProfile, project, location)
231 : undefined,
232 });
233
234 const profileIdOf = (value: string | undefined) => lastSegment(value ?? "");
235
236 const toAttrs = (item: ds.Stream, project: string, region: string) => {
237 const name = item.name ?? "";
238 const parsed = parseName(name, "streams", region);
239 return {
240 name,
241 streamId: parsed.id,
242 project: parsed.project || project,
243 location: parsed.location,
244 displayName: item.displayName,
245 labels: userLabels(item.labels),
246 sourceConfig: item.sourceConfig,
247 destinationConfig: item.destinationConfig,
248 backfillAll: item.backfillAll,
249 backfillNone: emptyMessage(item.backfillNone),
250 customerManagedEncryptionKey: item.customerManagedEncryptionKey,
251 ruleSets: item.ruleSets,
252 state: item.state,
253 errors: item.errors,
254 lastRecoveryTime: item.lastRecoveryTime,
255 satisfiesPzi: item.satisfiesPzi,
256 satisfiesPzs: item.satisfiesPzs,
257 createTime: item.createTime,
258 updateTime: item.updateTime,
259 };
260 };
261
262 const getByName = (name: string) =>
263 name.length === 0
264 ? Effect.undefined
265 : ds
266 .getProjectsLocationsStreams({ name })
267 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
268
269 const listOwned = (project: string, region: string) =>
270 listAtLocation(project, region, (parent) =>
271 collectPages(
272 ds.listProjectsLocationsStreams.pages({
273 parent,
274 pageSize: 1000,
275 }),
276 (page) => page.streams,
277 ).pipe(Effect.map((items) => items.filter((item) => hasAlchemyLabelMap(item.labels)))),
278 );
279
280 const sourceConfigWithoutProfile = (config: SourceConfig | undefined) => {
281 if (config === undefined) return undefined;
282 const { sourceConnectionProfile: _ignored, ...rest } = config;
283 return rest;
284 };
285
286 const destinationConfigWithoutProfile = (config: DestinationConfig | undefined) => {
287 if (config === undefined) return undefined;
288 const { destinationConnectionProfile: _ignored, ...rest } = config;
289 return rest;
290 };
291
292 export const StreamProvider = () =>
293 Provider.succeed(Stream, {
294 stables: ["name", "streamId", "project", "location", "createTime"],
295
296 diff: Effect.fn(function* ({ news, olds, output }) {
297 if (!isResolved(news)) return undefined;
298 const env = yield* GcpEnvironment.current;
299 const previousSource = profileIdOf(
300 olds?.sourceConfig?.sourceConnectionProfile ??
301 output?.sourceConfig?.sourceConnectionProfile,
302 );
303 const nextSource = profileIdOf(
304 news.sourceConfig.sourceConnectionProfile ??
305 olds?.sourceConfig?.sourceConnectionProfile ??
306 output?.sourceConfig?.sourceConnectionProfile,
307 );
308 const previousDest = profileIdOf(
309 olds?.destinationConfig?.destinationConnectionProfile ??
310 output?.destinationConfig?.destinationConnectionProfile,
311 );
312 const nextDest = profileIdOf(
313 news.destinationConfig.destinationConnectionProfile ??
314 olds?.destinationConfig?.destinationConnectionProfile ??
315 output?.destinationConfig?.destinationConnectionProfile,
316 );
317 const previousCmek =
318 olds?.customerManagedEncryptionKey ?? output?.customerManagedEncryptionKey;
319 const nextCmek = news.customerManagedEncryptionKey ?? previousCmek;
320 return replaceOnIdentity({
321 previousId: olds?.streamId ?? output?.streamId,
322 nextId: news.streamId ?? olds?.streamId ?? output?.streamId,
323 previousLocation: normalizeLocation(olds?.location ?? output?.location, env.region),
324 nextLocation: normalizeLocation(
325 news.location ?? olds?.location ?? output?.location,
326 env.region,
327 ),
328 extra:
329 (previousSource.length > 0 && nextSource.length > 0 && previousSource !== nextSource) ||
330 (previousDest.length > 0 && nextDest.length > 0 && previousDest !== nextDest) ||
331 (previousCmek !== undefined && nextCmek !== undefined && previousCmek !== nextCmek) ||
332 (previousCmek === undefined && nextCmek !== undefined),
333 });
334 }),
335
336 read: Effect.fn(function* ({ id, olds, output }) {
337 const env = yield* GcpEnvironment.current;
338 const streamId = yield* toPhysicalId(id, olds?.streamId, output?.streamId, "stream");
339 const location = normalizeLocation(olds?.location ?? output?.location, env.region);
340 const name = output?.name ?? resourceName(env.project, location, streamId);
341 const existing = yield* getByName(name);
342 if (existing === undefined) return undefined;
343 const attrs = toAttrs(existing, env.project, env.region);
344 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
345 }),
346
347 list: () =>
348 Effect.gen(function* () {
349 const env = yield* GcpEnvironment.current;
350 const items = yield* listOwned(env.project, env.region);
351 return items.map((item) => toAttrs(item, env.project, env.region));
352 }),
353
354 reconcile: Effect.fn(function* ({ id, news, output }) {
355 const env = yield* GcpEnvironment.current;
356 const streamId = yield* toPhysicalId(id, news.streamId, output?.streamId, "stream");
357 const location = normalizeLocation(news.location ?? output?.location, env.region);
358 const name = resourceName(env.project, location, streamId);
359 const desiredLabels = {
360 ...toLabels(news.labels),
361 ...(yield* createInternalLabels(id)),
362 };
363 const displayName = news.displayName ?? streamId;
364 const sourceConfig = expandSource(news.sourceConfig, env.project, location);
365 const destinationConfig = expandDestination(news.destinationConfig, env.project, location);
366
367 let current = yield* getByName(output?.name ?? name);
368
369 const backfillAll =
370 news.backfillAll !== undefined
371 ? news.backfillAll
372 : news.backfillNone !== undefined
373 ? undefined
374 : current?.backfillAll;
375 const backfillNone =
376 news.backfillNone !== undefined
377 ? {}
378 : news.backfillAll !== undefined
379 ? undefined
380 : (emptyMessage(current?.backfillNone) ??
381 (current?.backfillAll === undefined ? {} : undefined));
382 const body: ds.Stream = {
383 displayName,
384 labels: desiredLabels,
385 sourceConfig,
386 destinationConfig,
387 backfillAll,
388 backfillNone: backfillAll === undefined ? backfillNone : undefined,
389 customerManagedEncryptionKey: news.customerManagedEncryptionKey,
390 ruleSets: news.ruleSets ? [...news.ruleSets] : undefined,
391 state: news.state,
392 };
393
394 if (current === undefined) {
395 const created = yield* ds
396 .createProjectsLocationsStreams({
397 parent: locationParent(env.project, location),
398 streamId,
399 force: news.force,
400 validateOnly: news.validateOnly,
401 body,
402 })
403 .pipe(Effect.catchTag("Conflict", () => Effect.undefined));
404 yield* settleOperation(created);
405 current = yield* waitUntilExists(getByName(name), name);
406 }
407
408 if (current === undefined) {
409 return yield* new ResourceNotResolved({ resourceName: name });
410 }
411
412 const observedLabels = tagRecord(current.labels);
413 const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
414 const labelsChanged = upsert.length > 0 || removed.length > 0;
415 const displayNameChanged = (current.displayName ?? "") !== displayName;
416 const sourceChanged =
417 fingerprint(sourceConfigWithoutProfile(current.sourceConfig)) !==
418 fingerprint(sourceConfigWithoutProfile(sourceConfig));
419 const destinationChanged =
420 fingerprint(destinationConfigWithoutProfile(current.destinationConfig)) !==
421 fingerprint(destinationConfigWithoutProfile(destinationConfig));
422 const backfillChanged =
423 fingerprint({
424 backfillAll: current.backfillAll,
425 backfillNone: emptyMessage(current.backfillNone),
426 }) !==
427 fingerprint({
428 backfillAll,
429 backfillNone: backfillAll === undefined ? backfillNone : undefined,
430 });
431 const ruleSetsChanged = fingerprint(current.ruleSets) !== fingerprint(news.ruleSets);
432 const stateChanged = news.state !== undefined && (current.state ?? "") !== news.state;
433 const mask = fieldMask([
434 labelsChanged && "labels",
435 displayNameChanged && "displayName",
436 sourceChanged && "sourceConfig",
437 destinationChanged && "destinationConfig",
438 backfillChanged && (backfillAll !== undefined ? "backfillAll" : "backfillNone"),
439 ruleSetsChanged && "ruleSets",
440 stateChanged && "state",
441 ]);
442
443 if (mask.length > 0) {
444 const patch: ds.Stream = { name: current.name ?? name };
445 if (labelsChanged) patch.labels = desiredLabels;
446 if (displayNameChanged) patch.displayName = displayName;
447 if (sourceChanged) patch.sourceConfig = sourceConfig;
448 if (destinationChanged) patch.destinationConfig = destinationConfig;
449 if (backfillChanged) {
450 patch.backfillAll = backfillAll;
451 patch.backfillNone = backfillAll === undefined ? backfillNone : undefined;
452 }
453 if (ruleSetsChanged) {
454 patch.ruleSets = news.ruleSets ? [...news.ruleSets] : undefined;
455 }
456 if (stateChanged) patch.state = news.state;
457 const operation = yield* ds.patchProjectsLocationsStreams({
458 name: current.name ?? name,
459 updateMask: mask,
460 force: news.force,
461 validateOnly: news.validateOnly,
462 body: patch,
463 });
464 yield* settleOperation(operation);
465 current = yield* waitUntilExists(getByName(current.name ?? name), current.name ?? name);
466 }
467
468 return toAttrs(current, env.project, env.region);
469 }),
470
471 delete: Effect.fn(function* ({ output }) {
472 const operation = yield* ds
473 .deleteProjectsLocationsStreams({
474 name: output.name,
475 })
476 .pipe(
477 Effect.retry({
478 while: (error) => error._tag === "Conflict",
479 times: 8,
480 schedule: Schedule.spaced("2 seconds"),
481 }),
482 Effect.catchTag(["NotFound", "Conflict"], () => Effect.undefined),
483 );
484 // Stream deletes take a few minutes.
485 yield* settleOperation(operation, { notFoundOk: true });
486 yield* waitUntilGone(getByName(output.name), output.name, {
487 times: 15,
488 interval: "4 seconds",
489 });
490 }),
491 });