@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 372 lines 11.7 KB
Raw
1 import * as biglake from "@distilled.cloud/gcp/biglake_v1";
2 import { unownedUnlessPrevious } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import { tagRecord } from "alchemy/Tags";
7 import * as Effect from "effect/Effect";
8 import { GcpEnvironment } from "../Environment.ts";
9 import type { Providers } from "../Providers.ts";
10 import {
11 BiglakeNotResolved,
12 listLocations,
13 createInternalLabels,
14 expandDatabase,
15 hasAlchemyLabelMap,
16 hasAlchemyLabels,
17 ignoreGone,
18 listCatalogs,
19 listChildResources,
20 listDatabases,
21 listTables,
22 locationParent,
23 mergeParameters,
24 missingGet,
25 namedOf,
26 normalizeLocation,
27 parseResourceName,
28 replaceOnIdentity,
29 retryTransient,
30 sameJson,
31 sameText,
32 toPhysicalId,
33 updateMaskOf,
34 userLabels,
35 waitUntilGone,
36 } from "./internal.ts";
37
38 export type SerDeInfo = {
39 /**
40 * Fully qualified Java class name of the serialization library.
41 */
42 serializationLib?: string;
43 };
44
45 export type StorageDescriptor = {
46 /**
47 * Cloud Storage folder URI where the table data is stored, starting
48 * with `gs://`.
49 */
50 locationUri?: string;
51 /**
52 * Fully qualified Java class name of the input format.
53 */
54 inputFormat?: string;
55 /**
56 * Fully qualified Java class name of the output format.
57 */
58 outputFormat?: string;
59 /**
60 * Serializer and deserializer information.
61 */
62 serdeInfo?: SerDeInfo;
63 };
64
65 export type HiveTableOptions = {
66 /**
67 * Hive table type (`MANAGED_TABLE`, `EXTERNAL_TABLE`, …).
68 */
69 tableType?: string;
70 /**
71 * User-supplied Hive table parameters. Alchemy ownership keys
72 * (`alchemy-stack`, `alchemy-stage`, `alchemy-id`) are merged in
73 * automatically — BigLake tables have no labels field.
74 */
75 parameters?: Record<string, string>;
76 /**
77 * Physical storage information of the data.
78 */
79 storageDescriptor?: StorageDescriptor;
80 };
81
82 export type CatalogsDatabasesTableProps = {
83 /**
84 * Parent database. Full name
85 * `projects/{project}/locations/{location}/catalogs/{catalog}/databases/{database}`
86 * or the database id (combined with `catalog` and `location`).
87 * Immutable — changing it replaces the table.
88 */
89 database: string;
90 /**
91 * Parent catalog used when `database` is a bare id.
92 */
93 catalog?: string;
94 /**
95 * Table id (the `{table}` segment). If omitted, a unique id is
96 * generated. Immutable — changing it replaces the table.
97 */
98 tableId?: string;
99 /**
100 * Location used when `database` is a bare id.
101 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
102 */
103 location?: string;
104 /**
105 * Table type.
106 * @default "HIVE"
107 */
108 type?: biglake.TableTypeEnum | (string & {});
109 /**
110 * Hive table options. Required when `type` is `HIVE`.
111 */
112 hiveOptions?: HiveTableOptions;
113 };
114
115 export type CatalogsDatabasesTable = Resource<
116 "GCP.BigLake.CatalogsDatabasesTable",
117 CatalogsDatabasesTableProps,
118 {
119 /** Full resource name. */
120 name: string;
121 /** Table id (last path segment). */
122 tableId: string;
123 /** Parent database resource name. */
124 database: string;
125 /** Parent catalog resource name. */
126 catalog: string;
127 /** Project id. */
128 project: string;
129 /** Location id. */
130 location: string;
131 /** Table type. */
132 type: string | undefined;
133 /** Hive table type (`MANAGED_TABLE`, `EXTERNAL_TABLE`, …). */
134 tableType: string | undefined;
135 /** User Hive parameters (Alchemy ownership keys stripped). */
136 parameters: Record<string, string>;
137 /** Physical storage descriptor. */
138 storageDescriptor: StorageDescriptor | undefined;
139 /** Server checksum used for optimistic updates. */
140 etag: string | undefined;
141 /** RFC3339 creation timestamp. */
142 createTime: string | undefined;
143 /** RFC3339 last-update timestamp. */
144 updateTime: string | undefined;
145 /** RFC3339 deletion timestamp, if soft-deleted. */
146 deleteTime: string | undefined;
147 /** RFC3339 expire timestamp after delete. */
148 expireTime: string | undefined;
149 },
150 never,
151 Providers
152 >;
153
154 /**
155 * A Hive table inside a BigLake Metastore database.
156 *
157 * Tables have no labels field — Alchemy stamps ownership into
158 * `hiveOptions.parameters` so `list` / nuke can find them. Database,
159 * table id, and type are immutable. Hive options update in place.
160 *
161 * ### Creating a Table
162 * **Example:** Managed Hive table
163 * ```typescript
164 * const table = yield* GCP.BigLake.CatalogsDatabasesTable("Events", {
165 * database: database.name,
166 * hiveOptions: {
167 * tableType: "MANAGED_TABLE",
168 * storageDescriptor: {
169 * locationUri: `gs://${bucket.bucketName}/events`,
170 * inputFormat: "org.apache.hadoop.mapred.SequenceFileInputFormat",
171 * outputFormat:
172 * "org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat",
173 * },
174 * parameters: { owner: "analytics" },
175 * },
176 * });
177 * ```
178 *
179 * **Example:** Named external table
180 * ```typescript
181 * const table = yield* GCP.BigLake.CatalogsDatabasesTable("Events", {
182 * database: database.name,
183 * tableId: "page_views",
184 * hiveOptions: {
185 * tableType: "EXTERNAL_TABLE",
186 * storageDescriptor: {
187 * locationUri: "gs://my-bucket/page_views",
188 * },
189 * },
190 * });
191 * ```
192 *
193 * @resource
194 * @category BigLake
195 */
196 export const CatalogsDatabasesTable = Resource<CatalogsDatabasesTable>(
197 "GCP.BigLake.CatalogsDatabasesTable",
198 );
199
200 const databaseOf = (database: string, project: string, location: string, catalog?: string) =>
201 expandDatabase(database, project, location, catalog);
202
203 const resourceName = (database: string, tableId: string) => `${database}/tables/${tableId}`;
204
205 const storageDescriptorOf = (
206 descriptor: biglake.StorageDescriptor | undefined,
207 ): StorageDescriptor | undefined => {
208 if (descriptor === undefined) return undefined;
209 return {
210 locationUri: descriptor.locationUri,
211 inputFormat: descriptor.inputFormat,
212 outputFormat: descriptor.outputFormat,
213 serdeInfo: descriptor.serdeInfo,
214 };
215 };
216
217 const toAttrs = (table: biglake.Table, project: string) => {
218 const name = table.name ?? "";
219 const parsed = parseResourceName(name, "tables");
220 const catalogParsed = parseResourceName(parsed.parent, "databases");
221 return {
222 name,
223 tableId: parsed.id,
224 database: parsed.parent,
225 catalog: catalogParsed.parent,
226 project,
227 location: parsed.location,
228 type: table.type,
229 tableType: table.hiveOptions?.tableType,
230 parameters: userLabels(table.hiveOptions?.parameters),
231 storageDescriptor: storageDescriptorOf(table.hiveOptions?.storageDescriptor),
232 etag: table.etag,
233 createTime: table.createTime,
234 updateTime: table.updateTime,
235 deleteTime: table.deleteTime,
236 expireTime: table.expireTime,
237 };
238 };
239
240 const getByName = missingGet(biglake.getProjectsLocationsCatalogsDatabasesTables);
241
242 const desiredHiveOptions = (
243 news: CatalogsDatabasesTableProps,
244 parameters: Record<string, string>,
245 ): biglake.HiveTableOptions => ({
246 tableType: news.hiveOptions?.tableType,
247 parameters,
248 storageDescriptor: news.hiveOptions?.storageDescriptor,
249 });
250
251 export const CatalogsDatabasesTableProvider = () =>
252 Provider.succeed(CatalogsDatabasesTable, {
253 stables: ["name", "tableId", "database", "catalog", "project", "location", "createTime"],
254
255 diff: Effect.fn(function* ({ news, olds, output }) {
256 if (!isResolved(news)) return undefined;
257 const env = yield* GcpEnvironment.current;
258 const location = normalizeLocation(
259 news.location ?? olds?.location ?? output?.location,
260 env.region,
261 );
262 const typeChanged =
263 (olds?.type ?? output?.type) !== undefined &&
264 (news.type ?? "HIVE") !== (olds?.type ?? output?.type);
265 if (typeChanged) {
266 return { action: "replace" as const, deleteFirst: true };
267 }
268 return replaceOnIdentity({
269 previousId: olds?.tableId ?? output?.tableId,
270 nextId: news.tableId,
271 previousLocation: normalizeLocation(olds?.location ?? output?.location, env.region),
272 nextLocation: location,
273 previousParent: olds?.database ?? output?.database,
274 nextParent: databaseOf(news.database, env.project, location, news.catalog),
275 });
276 }),
277
278 read: Effect.fn(function* ({ id, olds, output, previous }) {
279 const env = yield* GcpEnvironment.current;
280 const location = normalizeLocation(olds?.location ?? output?.location, env.region);
281 const tableId = yield* toPhysicalId(id, olds?.tableId, output?.tableId);
282 const database =
283 olds?.database !== undefined
284 ? databaseOf(olds.database, env.project, location, olds.catalog)
285 : (output?.database ?? "");
286 const name = output?.name ?? (database.length > 0 ? resourceName(database, tableId) : "");
287 const existing = yield* getByName(name);
288 if (existing === undefined) return undefined;
289 const attrs = toAttrs(existing, env.project);
290 return (yield* hasAlchemyLabels(id, tagRecord(existing.hiveOptions?.parameters)))
291 ? attrs
292 : unownedUnlessPrevious(attrs, previous, "name");
293 }),
294
295 list: () =>
296 Effect.gen(function* () {
297 const env = yield* GcpEnvironment.current;
298 const catalogs = (yield* Effect.forEach(listLocations(env.region), (location) =>
299 listCatalogs(locationParent(env.project, location)),
300 )).flat();
301 const databases = yield* listChildResources(namedOf(catalogs), listDatabases);
302 const tables = yield* listChildResources(namedOf(databases), listTables);
303 return tables
304 .filter((item) => hasAlchemyLabelMap(item.hiveOptions?.parameters))
305 .map((item) => toAttrs(item, env.project));
306 }),
307
308 reconcile: Effect.fn(function* ({ id, news, output }) {
309 const env = yield* GcpEnvironment.current;
310 const location = normalizeLocation(news.location ?? output?.location, env.region);
311 const database = databaseOf(news.database, env.project, location, news.catalog);
312 const tableId = yield* toPhysicalId(id, news.tableId, output?.tableId);
313 const name = output?.name ?? resourceName(database, tableId);
314 const type = news.type ?? "HIVE";
315 const parameters = mergeParameters(
316 news.hiveOptions?.parameters,
317 yield* createInternalLabels(id),
318 );
319 const hiveOptions = desiredHiveOptions(news, parameters);
320
321 let current = yield* getByName(name);
322
323 if (current === undefined) {
324 const created = yield* retryTransient(
325 biglake.createProjectsLocationsCatalogsDatabasesTables({
326 parent: database,
327 tableId,
328 body: { type, hiveOptions },
329 }),
330 ).pipe(Effect.catchTag("Conflict", () => getByName(name)));
331 current = created ?? undefined;
332 }
333
334 if (current === undefined) {
335 return yield* new BiglakeNotResolved({ resourceName: name });
336 }
337
338 const currentName = current.name ?? name;
339 const typeChanged = !sameText(current.type, type);
340 const hiveChanged = !sameJson(current.hiveOptions, hiveOptions);
341
342 if (typeChanged || hiveChanged) {
343 current = yield* retryTransient(
344 biglake.patchProjectsLocationsCatalogsDatabasesTables({
345 name: currentName,
346 updateMask: updateMaskOf(
347 typeChanged ? "type" : undefined,
348 hiveChanged ? "hiveOptions" : undefined,
349 ),
350 body: {
351 name: currentName,
352 type,
353 hiveOptions,
354 etag: current.etag,
355 },
356 }),
357 );
358 }
359
360 return toAttrs(current, env.project);
361 }),
362
363 delete: Effect.fn(function* ({ output }) {
364 if (!output.name) return;
365 yield* ignoreGone(
366 biglake.deleteProjectsLocationsCatalogsDatabasesTables({
367 name: output.name,
368 }),
369 );
370 yield* waitUntilGone(getByName(output.name));
371 }),
372 });