@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 239 lines 7.88 KB
Raw
1 import * as dataplex from "@distilled.cloud/gcp/dataplex_v1";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import * as Data from "effect/Data";
7 import * as Effect from "effect/Effect";
8 import { GcpEnvironment } from "../Environment.ts";
9 import type { Providers } from "../Providers.ts";
10 import {
11 listChildResources,
12 listEntities,
13 listLakes,
14 listPartitions,
15 listZones,
16 parseResourceName,
17 replaceIfChanged,
18 hasAlchemyLabelMap,
19 } from "./shared.ts";
20
21 export type LakesEntitiesPartitionProps = {
22 /**
23 * Parent entity. Full name
24 * `projects/{project}/locations/{location}/lakes/{lake}/zones/{zone}/entities/{entity}`.
25 * Immutable — changing it replaces the partition.
26 */
27 entity: string;
28 /**
29 * Ordered partition values matching the parent entity's partition
30 * schema. Immutable — changing them replaces the partition.
31 */
32 values: string[];
33 /**
34 * Location of the partition data (`gs://bucket/path/key=value` or a
35 * BigQuery table). Immutable — changing it replaces the partition.
36 */
37 location: string;
38 };
39
40 export type LakesEntitiesPartition = Resource<
41 "GCP.Dataplex.LakesEntitiesPartition",
42 LakesEntitiesPartitionProps,
43 {
44 /** Full resource name `.../entities/{entity}/partitions/{values}`. */
45 name: string;
46 /** Parent entity resource name. */
47 entity: string;
48 /** Project id. */
49 project: string;
50 /** Region id. */
51 region: string;
52 /** Ordered partition values. */
53 values: string[];
54 /** Data location. */
55 location: string;
56 /** Server etag. */
57 etag: string | undefined;
58 },
59 never,
60 Providers
61 >;
62
63 /**
64 * A Dataplex metadata partition under a zone entity.
65 *
66 * Partitions have no labels or description field: `read` reports a
67 * partition it finds without prior state as unowned, and `list` returns
68 * the partitions of entities in Alchemy-labeled lakes. Identity is the parent entity plus `values`; changing
69 * `entity`, `values`, or `location` replaces the partition.
70 *
71 * ### Creating a Partition
72 * **Example:** Hive-style date partition
73 * ```typescript
74 * const partition = yield* GCP.Dataplex.LakesEntitiesPartition("Day", {
75 * entity: entity.name,
76 * values: ["2024-01-01"],
77 * location: `gs://${bucket.bucketName}/events/dt=2024-01-01`,
78 * });
79 * ```
80 *
81 * @resource
82 * @category Dataplex
83 */
84 export const LakesEntitiesPartition = Resource<LakesEntitiesPartition>(
85 "GCP.Dataplex.LakesEntitiesPartition",
86 );
87
88 export class LakesEntitiesPartitionNotResolved extends Data.TaggedError(
89 "GCP.Dataplex.LakesEntitiesPartitionNotResolved",
90 )<{
91 resourceName: string;
92 }> {}
93
94 const encodeValue = (value: string) => encodeURIComponent(encodeURIComponent(value));
95
96 const resourceNameOf = (entity: string, values: readonly string[]) =>
97 `${entity}/partitions/${values.map(encodeValue).join("/")}`;
98
99 const toAttrs = (
100 partition: dataplex.GoogleCloudDataplexV1Partition,
101 project: string,
102 entity: string,
103 ) => {
104 const name = partition.name ?? "";
105 const parsed = parseResourceName(name, "partitions");
106 return {
107 name,
108 entity: parsed.parent || entity,
109 project: parsed.project || project,
110 region: parsed.location,
111 values: partition.values ?? [],
112 location: partition.location ?? "",
113 etag: partition.etag,
114 };
115 };
116
117 const getByName = (name: string) =>
118 dataplex
119 .getProjectsLocationsLakesZonesEntitiesPartitions({ name })
120 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
121
122 const listOwnedEntityNames = (project: string, region: string) =>
123 Effect.gen(function* () {
124 const lakes = (yield* listLakes(project, region)).filter((lake) =>
125 hasAlchemyLabelMap(lake.labels),
126 );
127 const zones = yield* listChildResources(lakes, listZones);
128 const named = zones.filter((zone) => (zone.name ?? "").length > 0);
129 const tables = yield* Effect.forEach(named, (zone) => listEntities(zone.name!, "TABLES"), {
130 concurrency: 4,
131 });
132 const filesets = yield* Effect.forEach(named, (zone) => listEntities(zone.name!, "FILESETS"), {
133 concurrency: 4,
134 });
135 return [...tables.flat(), ...filesets.flat()]
136 .map((entity) => entity.name)
137 .filter((name): name is string => (name ?? "").length > 0);
138 });
139
140 export const LakesEntitiesPartitionProvider = () =>
141 Provider.succeed(LakesEntitiesPartition, {
142 stables: ["name", "entity", "project", "region", "values"],
143
144 diff: Effect.fn(function* ({ news, olds, output }) {
145 if (!isResolved(news)) return undefined;
146 const previousEntity = olds?.entity ?? output?.entity;
147 const nextEntity = news.entity ?? previousEntity;
148 const previousValues = olds?.values ?? output?.values;
149 const nextValues = news.values ?? previousValues;
150 const previousLocation = olds?.location ?? output?.location;
151 const nextLocation = news.location ?? previousLocation;
152 if (
153 replaceIfChanged(previousEntity, nextEntity) ||
154 (previousValues !== undefined &&
155 nextValues !== undefined &&
156 JSON.stringify(previousValues) !== JSON.stringify(nextValues)) ||
157 replaceIfChanged(previousLocation, nextLocation)
158 ) {
159 return {
160 action: "replace" as const,
161 deleteFirst:
162 previousEntity === nextEntity &&
163 previousLocation === nextLocation &&
164 previousValues !== undefined &&
165 nextValues !== undefined &&
166 JSON.stringify(previousValues) === JSON.stringify(nextValues),
167 };
168 }
169 return undefined;
170 }),
171
172 read: Effect.fn(function* ({ olds, output }) {
173 const env = yield* GcpEnvironment.current;
174 const entity = olds?.entity ?? output?.entity ?? "";
175 const values = olds?.values ?? output?.values ?? [];
176 const name = output?.name ?? (entity ? resourceNameOf(entity, values) : "");
177 if (name.length === 0) return undefined;
178 const existing = yield* getByName(name);
179 if (existing === undefined) return undefined;
180 const attrs = toAttrs(existing, env.project, entity);
181 // No labels field: without prior state the partition may not be ours.
182 return output !== undefined ? attrs : Unowned(attrs);
183 }),
184
185 list: () =>
186 Effect.gen(function* () {
187 const env = yield* GcpEnvironment.current;
188 const entities = yield* listOwnedEntityNames(env.project, env.region);
189 const groups = yield* Effect.forEach(
190 entities,
191 (entity) =>
192 listPartitions(entity).pipe(
193 Effect.map((partitions) =>
194 partitions.map((partition) => toAttrs(partition, env.project, entity)),
195 ),
196 ),
197 { concurrency: 4 },
198 );
199 return groups.flat();
200 }),
201
202 reconcile: Effect.fn(function* ({ news, output }) {
203 const env = yield* GcpEnvironment.current;
204 const entity = news.entity;
205 const name = output?.name ?? resourceNameOf(entity, news.values);
206 let current = yield* getByName(name);
207
208 if (current === undefined) {
209 const created = yield* dataplex
210 .createProjectsLocationsLakesZonesEntitiesPartitions({
211 parent: entity,
212 body: {
213 values: news.values,
214 location: news.location,
215 },
216 })
217 .pipe(Effect.catchTag("Conflict", () => getByName(name)));
218 current = created ?? undefined;
219 if (current === undefined) {
220 current = yield* getByName(name);
221 }
222 }
223
224 if (current === undefined) {
225 return yield* new LakesEntitiesPartitionNotResolved({ resourceName: name });
226 }
227
228 return toAttrs(current, env.project, entity);
229 }),
230
231 delete: Effect.fn(function* ({ output }) {
232 yield* dataplex
233 .deleteProjectsLocationsLakesZonesEntitiesPartitions({
234 name: output.name,
235 etag: output.etag,
236 })
237 .pipe(Effect.catchTag("NotFound", () => Effect.void));
238 }),
239 });