@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

main
ts 591 lines 20.1 KB
Raw
1 import {
2 diffMigrations,
3 migrationAttributesOf,
4 migrationsAttrs,
5 migrationsInputOf,
6 stampedOf,
7 type MigrationAttributes,
8 type MigrationsInput,
9 } from "@alchemy.run/sql/Migrations";
10 import * as spanner from "@distilled.cloud/gcp/spanner_v1";
11 import { isResolved } from "alchemy/Diff";
12 import { createPhysicalName } from "alchemy/PhysicalName";
13 import * as Provider from "alchemy/Provider";
14 import { Resource } from "alchemy/Resource";
15 import * as Data from "effect/Data";
16 import * as Effect from "effect/Effect";
17 import * as Schedule from "effect/Schedule";
18 import * as Stream from "effect/Stream";
19 import { GcpEnvironment } from "../Environment.ts";
20 import { ALCHEMY_LABEL_PREFIX } from "../Labels.ts";
21 import type { Providers } from "../Providers.ts";
22 import { runDatabaseMigrations } from "./Migrations.ts";
23 import { waitForOperation } from "./operations.ts";
24
25 const DEFAULT_DIALECT = "GOOGLE_STANDARD_SQL";
26 const MAX_DATABASE_ID_LENGTH = 30;
27
28 export type DatabaseDialect = spanner.DatabaseDatabaseDialectEnum | (string & {});
29
30 export type EncryptionConfig = {
31 /**
32 * Cloud KMS key
33 * (`projects/{project}/locations/{location}/keyRings/{keyRing}/cryptoKeys/{cryptoKey}`).
34 * Immutable — changing it replaces the database.
35 */
36 kmsKeyName?: string;
37 /**
38 * KMS keys covering every region of the instance config. Immutable —
39 * changing them replaces the database.
40 */
41 kmsKeyNames?: string[];
42 };
43
44 export type DatabaseProps = {
45 /**
46 * Parent instance id or full name
47 * (`projects/{project}/instances/{instance}`). Immutable — changing it
48 * replaces the database.
49 */
50 instance: string;
51 /**
52 * Database id (the `{database}` segment of
53 * `projects/{project}/instances/{instance}/databases/{database}`). If
54 * omitted, a unique name is generated from the stack, stage, and
55 * logical id. Must match `^[a-z][-a-z0-9]*[a-z0-9]$` (2–30
56 * characters). Immutable — changing it replaces the database.
57 */
58 databaseId?: string;
59 /**
60 * SQL dialect. Immutable — changing it replaces the database.
61 * @default "GOOGLE_STANDARD_SQL"
62 */
63 databaseDialect?: DatabaseDialect;
64 /**
65 * Extra DDL statements run atomically with create. Immutable —
66 * changing them replaces the database, so evolve a schema with
67 * `migrations` instead.
68 */
69 extraStatements?: string[];
70 /**
71 * SQL migrations to apply on deploy, once the database is ready. Accepts
72 * a directory path, a `{ dir, table }` object, or a `Drizzle.Schema`
73 * output. Only pending migrations run; editing or removing an applied
74 * file fails the deploy. Works on both dialects.
75 *
76 * Spanner can't commit DDL and DML together. A migration's DDL runs one
77 * statement per schema operation, each named after the migration and
78 * the statement: a deploy after a crash waits for the operations it
79 * started instead of rerunning them, and after a failed statement is
80 * fixed only that statement and the ones after it run. Its trailing DML
81 * commits with the bookkeeping row. DML between DDL statements fails the
82 * deploy; split it into its own migration. Bookkeeping lives in the `alchemy_migrations` table
83 * (Spanner names can't start with `_`).
84 */
85 migrations?: MigrationsInput;
86 /**
87 * Customer-managed encryption. Immutable — changing it replaces the
88 * database.
89 */
90 encryptionConfig?: EncryptionConfig;
91 /**
92 * Drop protection. Alchemy defaults to disabled so `destroy` can drop
93 * the database. Delete disables protection first when it is on.
94 * @default false
95 */
96 enableDropProtection?: boolean;
97 };
98
99 export type Database = Resource<
100 "GCP.Spanner.Database",
101 DatabaseProps,
102 {
103 /** Full resource name `projects/{project}/instances/{instance}/databases/{database}`. */
104 name: string;
105 /** Database id (last path segment). */
106 databaseId: string;
107 /** Parent instance id. */
108 instanceId: string;
109 /** Project id. */
110 project: string;
111 /** SQL dialect. */
112 databaseDialect: string | undefined;
113 /** Whether drop protection is enabled. */
114 enableDropProtection: boolean;
115 /** Current database state (`CREATING`, `READY`, `READY_OPTIMIZING`). */
116 state: string | undefined;
117 /** Version retention period (e.g. `"1h"`). */
118 versionRetentionPeriod: string | undefined;
119 /** Earliest PITR timestamp, if available. */
120 earliestVersionTime: string | undefined;
121 /** Default leader region, if set. */
122 defaultLeader: string | undefined;
123 /** CMEK config, if any. */
124 encryptionConfig: EncryptionConfig | undefined;
125 /** Whether an update is in flight. */
126 reconciling: boolean | undefined;
127 /** RFC3339 creation timestamp. */
128 createTime: string | undefined;
129 /** Directory of the applied SQL migrations. */
130 migrationsDir: string | undefined;
131 /** Migration bookkeeping table. */
132 migrationsTable: string | undefined;
133 /** Applied migration content hashes. */
134 migrationsHashes: Record<string, string>;
135 },
136 never,
137 Providers
138 >;
139
140 /**
141 * A Cloud Spanner database inside an instance.
142 *
143 * Spanner databases have no labels field. `list` enumerates databases on
144 * alchemy-labeled instances so `pnpm nuke:gcp` can find leaked rows.
145 * Changing `instance`, `databaseId`, dialect, CMEK, or extra DDL
146 * replaces the database. Drop protection updates in place.
147 *
148 * ### Creating a Database
149 * **Example:** Generated name on a Spanner instance
150 * ```typescript
151 * const instance = yield* GCP.Spanner.Instance("App", {});
152 * const database = yield* GCP.Spanner.Database("AppDb", {
153 * instance: instance.instanceId,
154 * });
155 * ```
156 *
157 * **Example:** Explicit id, dialect, and extra DDL
158 * ```typescript
159 * const database = yield* GCP.Spanner.Database("AppDb", {
160 * instance: instance.instanceId,
161 * databaseId: "appdb",
162 * extraStatements: [
163 * "CREATE TABLE Users (UserId INT64 NOT NULL) PRIMARY KEY (UserId)",
164 * ],
165 * });
166 * ```
167 *
168 * ### Querying
169 * **Example:** Run SQL
170 * ```typescript
171 * const executeSql = yield* GCP.Spanner.ExecuteSql(database);
172 * const result = yield* executeSql({ sql: "SELECT 1 AS n" });
173 * ```
174 *
175 * ### Reading Schema
176 * **Example:** Fetch live DDL
177 * ```typescript
178 * const getDdl = yield* GCP.Spanner.GetDdl(database);
179 * const { statements } = yield* getDdl();
180 * ```
181 *
182 * ### Migrations
183 * **Example:** Apply migrations on deploy
184 * ```typescript
185 * const database = yield* GCP.Spanner.Database("AppDb", {
186 * instance: instance.instanceId,
187 * migrations: "./migrations",
188 * });
189 * ```
190 *
191 * Each file in `./migrations` holds GoogleSQL (or PostgreSQL, with
192 * `databaseDialect: "POSTGRESQL"`) statements separated by `;`. Migrations
193 * run as the deploying identity, which needs `spanner.databases.updateDdl`
194 * and `spanner.databases.write`.
195 *
196 * @resource
197 * @category Spanner
198 */
199 export const Database = Resource<Database>("GCP.Spanner.Database");
200
201 export class DatabaseNotResolved extends Data.TaggedError("GCP.Spanner.DatabaseNotResolved")<{
202 resourceName: string;
203 }> {}
204
205 export class DatabaseNotReady extends Data.TaggedError("GCP.Spanner.DatabaseNotReady")<{
206 resourceName: string;
207 state: string;
208 }> {}
209
210 export class DatabaseStillExists extends Data.TaggedError("GCP.Spanner.DatabaseStillExists")<{
211 resourceName: string;
212 }> {}
213
214 const lastSegment = (value: string) => {
215 const trimmed = value.replace(/\/+$/, "");
216 const parts = trimmed.split("/");
217 return parts[parts.length - 1] || trimmed;
218 };
219
220 const instanceIdOf = (value: string) => lastSegment(value);
221
222 const normalizeDialect = (value: string | undefined) => {
223 const next = (value ?? DEFAULT_DIALECT).toUpperCase();
224 return next.endsWith("_UNSPECIFIED") ? DEFAULT_DIALECT : next;
225 };
226
227 const resourceName = (project: string, instanceId: string, databaseId: string) =>
228 `projects/${project}/instances/${instanceId}/databases/${databaseId}`;
229
230 const instanceName = (project: string, instanceId: string) =>
231 `projects/${project}/instances/${instanceId}`;
232
233 const parseName = (name: string) => {
234 const parts = name.split("/").filter((part) => part.length > 0);
235 const databasesAt = parts.lastIndexOf("databases");
236 const instancesAt = parts.lastIndexOf("instances");
237 const projectsAt = parts.lastIndexOf("projects");
238 return {
239 project: projectsAt >= 0 && parts[projectsAt + 1] ? parts[projectsAt + 1]! : "",
240 instanceId: instancesAt >= 0 && parts[instancesAt + 1] ? parts[instancesAt + 1]! : "",
241 databaseId:
242 databasesAt >= 0 && parts[databasesAt + 1] ? parts[databasesAt + 1]! : lastSegment(name),
243 };
244 };
245
246 const toSpannerId = (name: string) => {
247 let next = name
248 .toLowerCase()
249 .replace(/[^a-z0-9-]/g, "-")
250 .replace(/-+/g, "-");
251 next = next.replace(/^-+/, "").replace(/-+$/g, "");
252 if (!/^[a-z]/.test(next)) next = `d${next}`;
253 next = next.slice(0, MAX_DATABASE_ID_LENGTH).replace(/-+$/g, "");
254 if (next.length < 2) next = `${next}xx`.slice(0, MAX_DATABASE_ID_LENGTH);
255 return next;
256 };
257
258 const toDatabaseId = (id: string, databaseId: string | undefined, existing?: string) =>
259 Effect.gen(function* () {
260 if (databaseId !== undefined) return databaseId;
261 if (existing !== undefined) return existing;
262 return toSpannerId(
263 yield* createPhysicalName({
264 id,
265 maxLength: MAX_DATABASE_ID_LENGTH,
266 lowercase: true,
267 }),
268 );
269 });
270
271 const createStatementOf = (databaseId: string, dialect: string) =>
272 dialect === "POSTGRESQL"
273 ? `CREATE DATABASE "${databaseId}"`
274 : `CREATE DATABASE \`${databaseId}\``;
275
276 const encryptionOf = (
277 config: spanner.EncryptionConfig | undefined,
278 ): EncryptionConfig | undefined => {
279 if (config === undefined) return undefined;
280 if (
281 (config.kmsKeyName === undefined || config.kmsKeyName.length === 0) &&
282 (config.kmsKeyNames === undefined || config.kmsKeyNames.length === 0)
283 ) {
284 return undefined;
285 }
286 return {
287 kmsKeyName: config.kmsKeyName,
288 kmsKeyNames: config.kmsKeyNames,
289 };
290 };
291
292 const encryptionKey = (config: EncryptionConfig | undefined) =>
293 JSON.stringify({
294 kmsKeyName: config?.kmsKeyName ?? "",
295 kmsKeyNames: [...(config?.kmsKeyNames ?? [])].sort(),
296 });
297
298 const extraKey = (statements: string[] | undefined) => JSON.stringify(statements ?? []);
299
300 const toAttrs = (
301 database: spanner.Database,
302 project: string,
303 migrations?: Partial<MigrationAttributes>,
304 ): Database["Attributes"] => {
305 const name = database.name ?? "";
306 const parsed = parseName(name);
307 return {
308 name,
309 databaseId: parsed.databaseId,
310 instanceId: parsed.instanceId,
311 project: parsed.project || project,
312 databaseDialect: database.databaseDialect,
313 enableDropProtection: database.enableDropProtection === true,
314 state: database.state,
315 versionRetentionPeriod: database.versionRetentionPeriod,
316 earliestVersionTime: database.earliestVersionTime,
317 defaultLeader: database.defaultLeader,
318 encryptionConfig: encryptionOf(database.encryptionConfig),
319 reconciling: database.reconciling,
320 createTime: database.createTime,
321 ...migrationAttributesOf(migrations),
322 };
323 };
324
325 const getByName = (name: string) =>
326 spanner
327 .getProjectsInstancesDatabases({ name })
328 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
329
330 const waitUntilExists = (name: string) =>
331 getByName(name).pipe(
332 Effect.flatMap((database) =>
333 database
334 ? Effect.succeed(database)
335 : Effect.fail(new DatabaseNotResolved({ resourceName: name })),
336 ),
337 Effect.retry({
338 while: (error) => error._tag === "GCP.Spanner.DatabaseNotResolved",
339 times: 8,
340 schedule: Schedule.spaced("2 seconds"),
341 }),
342 );
343
344 const isReadyState = (state: string | undefined) =>
345 state === "READY" || state === "READY_OPTIMIZING";
346
347 const waitUntilReady = (name: string) =>
348 getByName(name).pipe(
349 Effect.filterOrFail(
350 (database): database is spanner.Database => database !== undefined,
351 () => new DatabaseNotResolved({ resourceName: name }),
352 ),
353 Effect.filterOrFail(
354 (database) => isReadyState(database.state),
355 (database) =>
356 new DatabaseNotReady({
357 resourceName: name,
358 state: database.state ?? "STATE_UNSPECIFIED",
359 }),
360 ),
361 Effect.retry({
362 while: (error) =>
363 error._tag === "GCP.Spanner.DatabaseNotReady" ||
364 error._tag === "GCP.Spanner.DatabaseNotResolved",
365 times: 10,
366 schedule: Schedule.spaced("8 seconds"),
367 }),
368 );
369
370 const waitUntilGone = (name: string) =>
371 getByName(name).pipe(
372 Effect.flatMap((database) =>
373 database === undefined
374 ? Effect.void
375 : Effect.fail(new DatabaseStillExists({ resourceName: name })),
376 ),
377 Effect.retry({
378 while: (error) => error._tag === "GCP.Spanner.DatabaseStillExists",
379 times: 10,
380 schedule: Schedule.spaced("8 seconds"),
381 }),
382 );
383
384 const retryConcurrentChanges = <A, E extends { readonly _tag: string }, R>(
385 effect: Effect.Effect<A, E, R>,
386 ) =>
387 effect.pipe(
388 Effect.retry({
389 while: (error) => error._tag === "Conflict",
390 times: 8,
391 schedule: Schedule.spaced("5 seconds"),
392 }),
393 );
394
395 const listAlchemyInstances = (project: string) =>
396 spanner.listProjectsInstances
397 .pages({
398 parent: `projects/${project}`,
399 pageSize: 1000,
400 })
401 .pipe(
402 Stream.flatMap((page) => Stream.fromIterable(page.instances ?? [])),
403 Stream.filter((instance) =>
404 Object.keys(instance.labels ?? {}).some((key) => key.startsWith(ALCHEMY_LABEL_PREFIX)),
405 ),
406 Stream.runCollect,
407 Effect.map((chunk) => Array.from(chunk)),
408 Effect.catchTag("NotFound", () => Effect.succeed([] as spanner.Instance[])),
409 );
410
411 export const DatabaseProvider = () =>
412 Provider.succeed(Database, {
413 stables: ["name", "databaseId", "instanceId", "project", "createTime"],
414
415 diff: Effect.fn(function* ({ news, olds, output }) {
416 if (!isResolved(news)) return undefined;
417
418 const previousInstance = olds?.instance ?? output?.instanceId;
419 const nextInstance = news.instance;
420 const previousId = olds?.databaseId ?? output?.databaseId;
421 const nextId = news.databaseId ?? previousId;
422 const previousDialect = normalizeDialect(olds?.databaseDialect ?? output?.databaseDialect);
423 const nextDialect = normalizeDialect(news.databaseDialect ?? output?.databaseDialect);
424 const previousKms = encryptionKey(olds?.encryptionConfig ?? output?.encryptionConfig);
425 const nextKms = encryptionKey(news.encryptionConfig ?? output?.encryptionConfig);
426 const extraChanged =
427 olds?.extraStatements !== undefined &&
428 news.extraStatements !== undefined &&
429 extraKey(olds.extraStatements) !== extraKey(news.extraStatements);
430
431 const instanceChanged =
432 previousInstance !== undefined &&
433 instanceIdOf(previousInstance) !== instanceIdOf(nextInstance);
434 const idChanged = previousId !== undefined && nextId !== undefined && previousId !== nextId;
435
436 const replace =
437 instanceChanged ||
438 idChanged ||
439 previousDialect !== nextDialect ||
440 previousKms !== nextKms ||
441 extraChanged;
442
443 if (!replace) {
444 if (yield* diffMigrations({ news, output })) {
445 return { action: "update" as const };
446 }
447 return undefined;
448 }
449 return {
450 action: "replace" as const,
451 deleteFirst: !instanceChanged && previousId !== undefined && nextId === previousId,
452 };
453 }),
454
455 read: Effect.fn(function* ({ id, olds, output }) {
456 const env = yield* GcpEnvironment.current;
457 const instanceId = instanceIdOf(olds?.instance ?? output?.instanceId ?? "");
458 if (instanceId.length === 0) return undefined;
459 const databaseId = yield* toDatabaseId(id, olds?.databaseId, output?.databaseId);
460 const name = output?.name ?? resourceName(env.project, instanceId, databaseId);
461 const existing = yield* getByName(name);
462 if (existing === undefined) return undefined;
463 return toAttrs(existing, env.project, output);
464 }),
465
466 list: () =>
467 Effect.gen(function* () {
468 const env = yield* GcpEnvironment.current;
469 const instances = yield* listAlchemyInstances(env.project);
470 const pages = yield* Effect.forEach(
471 instances,
472 (instance) => {
473 const parent = instance.name;
474 if (parent === undefined || parent.length === 0) {
475 return Effect.succeed([] as Database["Attributes"][]);
476 }
477 return spanner.listProjectsInstancesDatabases
478 .pages({
479 parent,
480 pageSize: 1000,
481 })
482 .pipe(
483 Stream.flatMap((page) => Stream.fromIterable(page.databases ?? [])),
484 Stream.map((database) => toAttrs(database, env.project)),
485 Stream.runCollect,
486 Effect.map((chunk) => Array.from(chunk)),
487 Effect.catchTag("NotFound", () => Effect.succeed([] as Database["Attributes"][])),
488 );
489 },
490 { concurrency: 4 },
491 );
492 return pages.flat();
493 }),
494
495 reconcile: Effect.fn(function* ({ id, news, output }) {
496 const env = yield* GcpEnvironment.current;
497 const instanceId = instanceIdOf(news.instance);
498 const databaseId = yield* toDatabaseId(id, news.databaseId, output?.databaseId);
499 const name = resourceName(env.project, instanceId, databaseId);
500 const dialect = normalizeDialect(news.databaseDialect);
501 const desiredProtection = news.enableDropProtection === true;
502
503 let current = yield* getByName(output?.name ?? name);
504
505 if (current === undefined) {
506 const created = yield* spanner
507 .createProjectsInstancesDatabases({
508 parent: instanceName(env.project, instanceId),
509 body: {
510 createStatement: createStatementOf(databaseId, dialect),
511 extraStatements: news.extraStatements,
512 encryptionConfig: news.encryptionConfig,
513 databaseDialect: dialect === DEFAULT_DIALECT ? undefined : dialect,
514 },
515 })
516 .pipe(Effect.catchTag("Conflict", () => Effect.undefined));
517 if (created !== undefined) {
518 yield* waitForOperation(created, { alreadyExistsOk: true });
519 }
520 current = yield* waitUntilExists(name);
521 }
522
523 if (current === undefined) {
524 return yield* new DatabaseNotResolved({ resourceName: name });
525 }
526
527 if (!isReadyState(current.state)) {
528 current = yield* waitUntilReady(name);
529 }
530
531 const observedProtection = current.enableDropProtection === true;
532 if (observedProtection !== desiredProtection) {
533 const patched = yield* retryConcurrentChanges(
534 spanner.patchProjectsInstancesDatabases({
535 name,
536 updateMask: "enableDropProtection",
537 body: {
538 name,
539 enableDropProtection: desiredProtection,
540 },
541 }),
542 );
543 yield* waitForOperation(patched);
544 current = yield* waitUntilReady(name);
545 }
546
547 // Sync migrations — the shared pipeline skips already-applied ones.
548 const migrationsInput = migrationsInputOf(news);
549 const migrations = migrationsInput
550 ? yield* runDatabaseMigrations({
551 database: current.name ?? name,
552 dialect:
553 normalizeDialect(current.databaseDialect) === "POSTGRESQL" ? "postgres" : "googlesql",
554 input: migrationsInput,
555 stamped: stampedOf(output),
556 })
557 : undefined;
558
559 return toAttrs(
560 current,
561 env.project,
562 migrationsAttrs({ input: migrationsInput, run: migrations, output }),
563 );
564 }),
565
566 delete: Effect.fn(function* ({ output }) {
567 const current = yield* getByName(output.name);
568 if (current === undefined) return;
569
570 if (current.enableDropProtection === true) {
571 const patched = yield* retryConcurrentChanges(
572 spanner.patchProjectsInstancesDatabases({
573 name: output.name,
574 updateMask: "enableDropProtection",
575 body: {
576 name: output.name,
577 enableDropProtection: false,
578 },
579 }),
580 );
581 yield* waitForOperation(patched);
582 }
583
584 yield* retryConcurrentChanges(
585 spanner
586 .dropDatabaseProjectsInstancesDatabases({ database: output.name })
587 .pipe(Effect.catchTag("NotFound", () => Effect.void)),
588 );
589 yield* waitUntilGone(output.name);
590 }),
591 });