The best way to control Claude Code, Codex and more from any computer or phone

main
ts 522 lines 17.4 KB
Raw
1 import * as storage from "@distilled.cloud/gcp/storage_v1";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import { createPhysicalName } from "alchemy/PhysicalName";
5 import * as Provider from "alchemy/Provider";
6 import { Resource } from "alchemy/Resource";
7 import { tagRecord } from "alchemy/Tags";
8 import * as Data from "effect/Data";
9 import * as Effect from "effect/Effect";
10 import * as Schedule from "effect/Schedule";
11 import * as Stream from "effect/Stream";
12 import { GcpEnvironment } from "../Environment.ts";
13 import {
14 createInternalLabels,
15 diffLabels,
16 hasAlchemyLabels,
17 stripInternalLabels,
18 toLabels,
19 } from "../Labels.ts";
20 import type { Providers } from "../Providers.ts";
21
22 export type BucketProps = {
23 /**
24 * Globally-unique bucket name. If omitted, a unique name is generated from
25 * the stack, stage, and logical id. Must be 3-63 characters, lowercase,
26 * and DNS compatible.
27 */
28 bucketName?: string;
29 /**
30 * Location of the bucket (`US`, `EU`, `US-CENTRAL1`, …). Immutable —
31 * changing it replaces the bucket.
32 * @default the stack's GCP region (`GCP.Region`, else the profile region, else `us-central1`)
33 */
34 location?: string;
35 /**
36 * Default storage class for new objects.
37 * @default "STANDARD"
38 */
39 storageClass?: string;
40 /**
41 * Whether object versioning is enabled.
42 * @default false
43 */
44 versioning?: boolean;
45 /**
46 * User labels. Alchemy ownership labels are merged in automatically.
47 */
48 labels?: Record<string, string>;
49 /**
50 * Delete all objects (and versions) before destroying the bucket.
51 * @default false
52 */
53 forceDestroy?: boolean;
54 /**
55 * Enable uniform bucket-level access. Required for managed folders.
56 * Hierarchical namespace implies this is enabled. Fine-grained ACLs
57 * cannot be used when this is true.
58 * @default false
59 */
60 uniformBucketLevelAccess?: boolean;
61 /**
62 * Enable hierarchical namespace (first-class folders). Immutable —
63 * must be set at create time; changing it replaces the bucket.
64 * Implies uniform bucket-level access.
65 * @default false
66 */
67 hierarchicalNamespace?: boolean;
68 /**
69 * Static website configuration. Cloud Storage applies it when the bucket
70 * is served through a CNAME or an external HTTPS load balancer (backend
71 * bucket) — not on `storage.googleapis.com/{bucket}/…` path-style URLs.
72 * Omitting it removes the configuration.
73 */
74 website?: BucketWebsite;
75 };
76
77 export interface BucketWebsite {
78 /**
79 * Object suffix appended to directory-style requests, e.g. `index.html`
80 * so `/docs/` serves `docs/index.html`.
81 */
82 mainPageSuffix?: string;
83 /** Object served (with status 404) when a requested object is missing. */
84 notFoundPage?: string;
85 }
86
87 export type Bucket = Resource<
88 "GCP.Storage.Bucket",
89 BucketProps,
90 {
91 /** Globally unique bucket name. */
92 bucketName: string;
93 /** Location of the bucket. */
94 location: string;
95 /** Location type (`region`, `multi-region`, `dual-region`). */
96 locationType: string | undefined;
97 /** Default storage class. */
98 storageClass: string | undefined;
99 /** Whether object versioning is enabled. */
100 versioning: boolean;
101 /** User labels (Alchemy ownership labels stripped). */
102 labels: Record<string, string>;
103 /** GCS self-link. */
104 selfLink: string | undefined;
105 /** Numeric project that owns the bucket. */
106 projectNumber: string | undefined;
107 /** RFC3339 creation timestamp. */
108 timeCreated: string | undefined;
109 /** Whether uniform bucket-level access is enabled. */
110 uniformBucketLevelAccess: boolean;
111 /** Whether hierarchical namespace is enabled. */
112 hierarchicalNamespace: boolean;
113 /** Static website configuration, if set. */
114 website: BucketWebsite | undefined;
115 },
116 never,
117 Providers
118 >;
119
120 /**
121 * A Google Cloud Storage bucket.
122 *
123 * ### Creating a Bucket
124 * **Example:** Generated name
125 * ```typescript
126 * const bucket = yield* GCP.Storage.Bucket("assets", {
127 * forceDestroy: true,
128 * });
129 * ```
130 *
131 * **Example:** Explicit name, location, and labels
132 * ```typescript
133 * const bucket = yield* GCP.Storage.Bucket("assets", {
134 * bucketName: "my-app-assets",
135 * location: "US-CENTRAL1",
136 * storageClass: "STANDARD",
137 * versioning: true,
138 * labels: { env: "prod" },
139 * forceDestroy: true,
140 * });
141 * ```
142 *
143 * **Example:** Hierarchical namespace (folders)
144 * ```typescript
145 * const bucket = yield* GCP.Storage.Bucket("tree", {
146 * hierarchicalNamespace: true,
147 * uniformBucketLevelAccess: true,
148 * forceDestroy: true,
149 * });
150 * ```
151 *
152 * **Example:** Static website bucket with public read
153 * ```typescript
154 * const site = yield* GCP.Storage.Bucket("site", {
155 * uniformBucketLevelAccess: true,
156 * website: { mainPageSuffix: "index.html", notFoundPage: "404.html" },
157 * forceDestroy: true,
158 * });
159 * yield* GCP.IAM.Member("PublicRead", {
160 * kind: "storage.bucket",
161 * name: site.bucketName,
162 * role: "roles/storage.objectViewer",
163 * member: "allUsers",
164 * });
165 * ```
166 *
167 * ### Updating a Bucket
168 * **Example:** Change labels
169 * ```typescript
170 * const bucket = yield* GCP.Storage.Bucket("assets", {
171 * forceDestroy: true,
172 * labels: { env: "prod" },
173 * });
174 * ```
175 *
176 * ### Binding from a Function
177 * **Example:** PutObject from Cloud Run
178 * ```typescript
179 * export class Api extends GCP.Function<Api>()(
180 * "Api",
181 * { main: import.meta.url },
182 * Effect.gen(function* () {
183 * const bucket = yield* GCP.Storage.Bucket("assets", { forceDestroy: true });
184 * const putObject = yield* GCP.Storage.PutObject(bucket);
185 * return {
186 * fetch: Effect.gen(function* () {
187 * yield* putObject({ name: "hello.txt", body: "Hello!" }).pipe(
188 * Effect.orDie,
189 * );
190 * return HttpServerResponse.text("ok");
191 * }),
192 * };
193 * }).pipe(Effect.provide([GCP.Storage.PutObjectHttp])),
194 * ) {}
195 * ```
196 *
197 * ### Destroying a Bucket
198 * **Example:** Empty then delete
199 * ```typescript
200 * const bucket = yield* GCP.Storage.Bucket("assets", {
201 * forceDestroy: true,
202 * });
203 * ```
204 * `forceDestroy: true` deletes objects before the bucket. `alchemy destroy` waits until the bucket is gone.
205 *
206 * @resource
207 * @category Storage
208 */
209 export const Bucket = Resource<Bucket>("GCP.Storage.Bucket");
210
211 export class BucketNotResolved extends Data.TaggedError("GCP.Storage.BucketNotResolved")<{
212 bucketName: string;
213 }> {}
214
215 const userLabels = (
216 labels: Record<string, string | undefined> | null | undefined,
217 ): Record<string, string> => stripInternalLabels(tagRecord(labels));
218
219 const toName = (id: string, name: string | undefined, existing?: string) =>
220 Effect.gen(function* () {
221 return (
222 name ??
223 existing ??
224 (yield* createPhysicalName({
225 id,
226 maxLength: 63,
227 lowercase: true,
228 }))
229 );
230 });
231
232 const toAttrs = (bucket: storage.Bucket) => ({
233 bucketName: bucket.name ?? bucket.id ?? "",
234 location: bucket.location ?? "",
235 locationType: bucket.locationType,
236 storageClass: bucket.storageClass,
237 versioning: bucket.versioning?.enabled === true,
238 labels: userLabels(bucket.labels),
239 selfLink: bucket.selfLink,
240 projectNumber: bucket.projectNumber,
241 timeCreated: bucket.timeCreated,
242 uniformBucketLevelAccess: bucket.iamConfiguration?.uniformBucketLevelAccess?.enabled === true,
243 hierarchicalNamespace: bucket.hierarchicalNamespace?.enabled === true,
244 website: toWebsite(bucket.website),
245 });
246
247 const toWebsite = (website: storage.BucketWebsite | undefined): BucketWebsite | undefined =>
248 website === undefined ||
249 (website.mainPageSuffix === undefined && website.notFoundPage === undefined)
250 ? undefined
251 : {
252 mainPageSuffix: website.mainPageSuffix,
253 notFoundPage: website.notFoundPage,
254 };
255
256 const getByName = (bucketName: string) =>
257 storage
258 .getBuckets({ bucket: bucketName, projection: "full" })
259 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
260
261 const emptyBucket = (bucketName: string) =>
262 storage.listObjects.items({ bucket: bucketName, versions: true, maxResults: 1000 }).pipe(
263 Stream.runCollect,
264 Effect.flatMap((chunk) =>
265 Effect.forEach(
266 chunk,
267 (object) =>
268 object.name
269 ? storage
270 .deleteObjects({
271 bucket: bucketName,
272 object: object.name,
273 generation: object.generation,
274 })
275 .pipe(Effect.catchTag("NotFound", () => Effect.void))
276 : Effect.void,
277 { concurrency: 8 },
278 ),
279 ),
280 Effect.catchTag("NotFound", () => Effect.void),
281 );
282
283 const emptyFolders = (bucketName: string) =>
284 storage.listFolders.items({ bucket: bucketName, pageSize: 1000 }).pipe(
285 Stream.runCollect,
286 Effect.flatMap((chunk) => {
287 const folders = Array.from(chunk)
288 .map((folder) => folder.name)
289 .filter((name): name is string => !!name)
290 .sort((left, right) => right.length - left.length);
291 return Effect.forEach(
292 folders,
293 (folder) =>
294 storage
295 .deleteFolders({ bucket: bucketName, folder })
296 .pipe(Effect.catchTag("NotFound", () => Effect.void)),
297 { concurrency: 1 },
298 );
299 }),
300 Effect.catchTag("NotFound", () => Effect.void),
301 );
302
303 const emptyManagedFolders = (bucketName: string) =>
304 storage.listManagedFolders.items({ bucket: bucketName, pageSize: 1000 }).pipe(
305 Stream.runCollect,
306 Effect.flatMap((chunk) => {
307 const folders = Array.from(chunk)
308 .map((folder) => folder.name)
309 .filter((name): name is string => !!name)
310 .sort((left, right) => right.length - left.length);
311 return Effect.forEach(
312 folders,
313 (managedFolder) =>
314 storage
315 .deleteManagedFolders({
316 bucket: bucketName,
317 managedFolder,
318 allowNonEmpty: true,
319 })
320 .pipe(Effect.catchTag("NotFound", () => Effect.void)),
321 { concurrency: 1 },
322 );
323 }),
324 Effect.catchTag("NotFound", () => Effect.void),
325 );
326
327 export const BucketProvider = () =>
328 Provider.succeed(Bucket, {
329 stables: ["bucketName", "location", "locationType", "projectNumber", "timeCreated", "selfLink"],
330
331 diff: Effect.fn(function* ({ news, olds, output }) {
332 if (!isResolved(news)) return undefined;
333 // Bucket names are immutable: a new name is a new bucket, and the
334 // old one must be deleted rather than left behind.
335 const recordedName = output?.bucketName ?? olds?.bucketName;
336 if (
337 news.bucketName !== undefined &&
338 recordedName !== undefined &&
339 news.bucketName !== recordedName
340 ) {
341 return { action: "replace" as const, deleteFirst: false };
342 }
343 const previous = olds?.location ?? output?.location;
344 const next = news.location;
345 if (
346 previous !== undefined &&
347 next !== undefined &&
348 previous.toUpperCase() !== next.toUpperCase()
349 ) {
350 const previousName = olds?.bucketName ?? output?.bucketName;
351 const nextName = news.bucketName ?? previousName;
352 return {
353 action: "replace" as const,
354 deleteFirst: nextName !== undefined && nextName === previousName,
355 };
356 }
357 const previousHns = olds?.hierarchicalNamespace ?? output?.hierarchicalNamespace;
358 const nextHns = news.hierarchicalNamespace === true;
359 if (previousHns !== undefined && previousHns !== nextHns) {
360 const previousName = olds?.bucketName ?? output?.bucketName;
361 const nextName = news.bucketName ?? previousName;
362 return {
363 action: "replace" as const,
364 deleteFirst: nextName !== undefined && nextName === previousName,
365 };
366 }
367 return undefined;
368 }),
369
370 read: Effect.fn(function* ({ id, olds, output }) {
371 const bucketName = yield* toName(id, olds?.bucketName, output?.bucketName);
372 const existing = yield* getByName(bucketName);
373 if (existing === undefined) return undefined;
374 const attrs = toAttrs(existing);
375 return (yield* hasAlchemyLabels(id, tagRecord(existing.labels))) ? attrs : Unowned(attrs);
376 }),
377
378 list: () =>
379 Effect.gen(function* () {
380 const env = yield* GcpEnvironment.current;
381 return yield* storage.listBuckets.items({ project: env.project, projection: "full" }).pipe(
382 Stream.filter((bucket) =>
383 Object.keys(bucket.labels ?? {}).some((key) => key.startsWith("alchemy-")),
384 ),
385 Stream.map(toAttrs),
386 Stream.runCollect,
387 Effect.map((chunk) => Array.from(chunk)),
388 );
389 }),
390
391 reconcile: Effect.fn(function* ({ id, news, output }) {
392 const env = yield* GcpEnvironment.current;
393 const bucketName = yield* toName(id, news.bucketName, output?.bucketName);
394 const location = news.location ?? output?.location ?? env.region.toUpperCase();
395 const storageClass = news.storageClass ?? "STANDARD";
396 const versioning = news.versioning === true;
397 const hierarchicalNamespace = news.hierarchicalNamespace === true;
398 const uniformBucketLevelAccess =
399 news.uniformBucketLevelAccess === true || hierarchicalNamespace;
400 const configureIam = news.uniformBucketLevelAccess !== undefined || hierarchicalNamespace;
401 const desiredLabels = {
402 ...toLabels(news.labels),
403 ...(yield* createInternalLabels(id)),
404 };
405 const desiredWebsite = toWebsite(news.website);
406
407 let current = yield* getByName(bucketName);
408
409 if (current === undefined) {
410 const created = yield* storage
411 .insertBuckets({
412 project: env.project,
413 projection: "full",
414 body: {
415 name: bucketName,
416 location,
417 storageClass,
418 versioning: { enabled: versioning },
419 labels: desiredLabels,
420 hierarchicalNamespace: hierarchicalNamespace ? { enabled: true } : undefined,
421 website: desiredWebsite,
422 iamConfiguration: configureIam
423 ? {
424 uniformBucketLevelAccess: {
425 enabled: uniformBucketLevelAccess,
426 },
427 }
428 : undefined,
429 },
430 })
431 .pipe(Effect.catchTag("Conflict", () => getByName(bucketName)));
432 current = created ?? undefined;
433 }
434
435 if (current === undefined) {
436 return yield* new BucketNotResolved({ bucketName });
437 }
438
439 const observedLabels = tagRecord(current.labels);
440 const { upsert, removed } = diffLabels(observedLabels, desiredLabels);
441 const labelsChanged = upsert.length > 0 || removed.length > 0;
442 const storageClassChanged = (current.storageClass ?? "STANDARD") !== storageClass;
443 const versioningChanged = (current.versioning?.enabled === true) !== versioning;
444 const ublChanged =
445 configureIam &&
446 (current.iamConfiguration?.uniformBucketLevelAccess?.enabled === true) !==
447 uniformBucketLevelAccess;
448 const observedWebsite = toWebsite(current.website);
449 const websiteChanged =
450 observedWebsite?.mainPageSuffix !== desiredWebsite?.mainPageSuffix ||
451 observedWebsite?.notFoundPage !== desiredWebsite?.notFoundPage;
452
453 if (
454 labelsChanged ||
455 storageClassChanged ||
456 versioningChanged ||
457 ublChanged ||
458 websiteChanged
459 ) {
460 const nextLabels: Record<string, string | null> = { ...desiredLabels };
461 for (const key of removed) {
462 nextLabels[key] = null;
463 }
464 current = yield* storage.patchBuckets({
465 bucket: bucketName,
466 projection: "full",
467 body: {
468 storageClass,
469 versioning: { enabled: versioning },
470 labels: nextLabels as unknown as Record<string, string>,
471 iamConfiguration: ublChanged
472 ? {
473 uniformBucketLevelAccess: {
474 enabled: uniformBucketLevelAccess,
475 },
476 }
477 : undefined,
478 // PATCH merges nested objects and clears a field only when it
479 // is sent as JSON null.
480 website: websiteChanged
481 ? ((desiredWebsite === undefined
482 ? null
483 : {
484 mainPageSuffix: desiredWebsite.mainPageSuffix ?? null,
485 notFoundPage: desiredWebsite.notFoundPage ?? null,
486 }) as unknown as storage.BucketWebsite)
487 : undefined,
488 },
489 });
490 }
491
492 return toAttrs(current);
493 }),
494
495 delete: Effect.fn(function* ({ olds, output, force }) {
496 const mayEmpty = olds.forceDestroy === true || force === true;
497 const empty = Effect.gen(function* () {
498 yield* emptyBucket(output.bucketName);
499 if (output.hierarchicalNamespace) {
500 yield* emptyFolders(output.bucketName);
501 }
502 if (output.uniformBucketLevelAccess) {
503 yield* emptyManagedFolders(output.bucketName);
504 }
505 });
506 // A writer that is still running (e.g. a Dataflow job staging temp
507 // files) can add objects between the empty and the delete, which fails
508 // with 409 "not empty"; with forceDestroy, empty again and retry.
509 yield* Effect.gen(function* () {
510 if (mayEmpty) yield* empty;
511 yield* storage
512 .deleteBuckets({ bucket: output.bucketName })
513 .pipe(Effect.catchTag("NotFound", () => Effect.void));
514 }).pipe(
515 Effect.retry({
516 while: (error) => mayEmpty && error._tag === "Conflict",
517 times: 6,
518 schedule: Schedule.spaced("10 seconds"),
519 }),
520 );
521 }),
522 });