The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import * as crypto from "node:crypto"; |
| 2 | import type { Credentials } from "@distilled.cloud/gcp/Credentials"; |
| 3 | import * as storage from "@distilled.cloud/gcp/storage_v1"; |
| 4 | import { CredentialsStoreLive } from "alchemy/Auth/Credentials"; |
| 5 | import { type BucketObjects, makeBucketState } from "alchemy/State/BucketState"; |
| 6 | import * as Provisioning from "alchemy/State/Provisioning"; |
| 7 | import { State, StateStoreError } from "alchemy/State/State"; |
| 8 | import { recordStateStoreInit } from "alchemy/Telemetry/Metrics"; |
| 9 | import { globalLogs } from "alchemy/Util/LogRecord"; |
| 10 | import * as Effect from "effect/Effect"; |
| 11 | import type { HttpClient } from "effect/http/HttpClient"; |
| 12 | import * as Layer from "effect/Layer"; |
| 13 | import * as Schedule from "effect/Schedule"; |
| 14 | import * as Scope from "effect/Scope"; |
| 15 | import * as Stream from "effect/Stream"; |
| 16 | import { GcpAuth } from "../AuthProvider.ts"; |
| 17 | import * as GcpCredentials from "../Credentials.ts"; |
| 18 | import { fromCredentials, GcpEnvironment } from "../Environment.ts"; |
| 19 | import { makeObjectMedia } from "../Storage/ObjectMedia.ts"; |
| 20 | |
| 21 | export interface GcsStateOptions { |
| 22 | /** |
| 23 | * A store other than the project's default, by logical name: its bucket is |
| 24 | * `alchemy-state-<name>-<project>`. The bucket is created on first use, |
| 25 | * except in CI, where `alchemy state bootstrap --backend gcp --name <name>` |
| 26 | * provisions it ahead. |
| 27 | * |
| 28 | * @default the project's default store, `alchemy-state-<project>` |
| 29 | */ |
| 30 | name?: string; |
| 31 | /** |
| 32 | * A full bucket name instead of one derived from {@link GcsStateOptions.name}, |
| 33 | * e.g. an existing bucket. Created on first use if it does not exist. |
| 34 | */ |
| 35 | bucketName?: string; |
| 36 | /** |
| 37 | * Key prefix within the bucket under which all state objects are stored, |
| 38 | * e.g. `"alchemy"`. A trailing `/` is appended automatically. |
| 39 | * |
| 40 | * @default "" (bucket root) |
| 41 | */ |
| 42 | prefix?: string; |
| 43 | /** |
| 44 | * Location of a bucket the store creates, e.g. `"us-central1"` or the |
| 45 | * multi-region `"US"`. An existing bucket keeps its location. |
| 46 | * |
| 47 | * @default the profile's region, else `us-central1` |
| 48 | */ |
| 49 | location?: string; |
| 50 | } |
| 51 | |
| 52 | /** Context required by the distilled Cloud Storage operations. */ |
| 53 | type GcsDeps = Credentials | HttpClient; |
| 54 | |
| 55 | /** Cloud Storage bucket names are at most 63 characters. */ |
| 56 | const BUCKET_NAME_MAX = 63; |
| 57 | |
| 58 | /** |
| 59 | * Build the default state bucket name, `alchemy-state-[<name>-]<project>`. |
| 60 | * Bucket names are global, and project ids are unique, so the name is |
| 61 | * deterministic per project. A name too long for Cloud Storage replaces the |
| 62 | * project id with a hash of it. |
| 63 | */ |
| 64 | export const createStateBucketName = (project: string, name?: string) => { |
| 65 | const base = `alchemy-state-${name === undefined ? "" : `${name}-`}`; |
| 66 | const full = `${base}${project}`.toLowerCase(); |
| 67 | if (full.length <= BUCKET_NAME_MAX) return full; |
| 68 | const hash = crypto.createHash("sha256").update(project).digest("hex").slice(0, 16); |
| 69 | return `${base}${hash}`.toLowerCase(); |
| 70 | }; |
| 71 | |
| 72 | /** Whether a state bucket exists. */ |
| 73 | export const stateBucketExists = (bucket: string) => |
| 74 | storage.getBuckets({ bucket }).pipe( |
| 75 | Effect.as(true), |
| 76 | Effect.catchTag("NotFound", () => Effect.succeed(false)), |
| 77 | ); |
| 78 | |
| 79 | /** |
| 80 | * State store backed by a Google Cloud Storage bucket. |
| 81 | * |
| 82 | * Stack state is persisted as JSON objects in a bucket in your project, |
| 83 | * laid out like the local state store's file tree: |
| 84 | * |
| 85 | * ``` |
| 86 | * gs://{bucket}/{prefix}{stack}/{stage}/{fqn}.json |
| 87 | * gs://{bucket}/{prefix}{stack}/{stage}/__stack_output__.json |
| 88 | * gs://{bucket}/{prefix}{stack}/{stage}/__lease__.json |
| 89 | * ``` |
| 90 | * |
| 91 | * Concurrent deploys of one stack and stage share an `__lease__.json` |
| 92 | * object written with `ifGenerationMatch` preconditions. The second deploy |
| 93 | * fails immediately instead of mixing rows. |
| 94 | * |
| 95 | * The bucket is created on the first state operation if it does not |
| 96 | * exist, with uniform bucket-level access, public access prevention, and |
| 97 | * object versioning. Project Owners and Editors get |
| 98 | * `roles/storage.objectAdmin` on it, since uniform access otherwise leaves |
| 99 | * them unable to read objects. Nothing touches GCP credentials at layer |
| 100 | * construction time. |
| 101 | * |
| 102 | * ### Using the Cloud Storage State Store |
| 103 | * Pass `GCP.state()` as the `state` option of a Stack. By default the state |
| 104 | * is stored in a bucket named `alchemy-state-{project}`. |
| 105 | * |
| 106 | * **Example:** Default bucket |
| 107 | * ```typescript |
| 108 | * import * as Alchemy from "alchemy"; |
| 109 | * import * as GCP from "@alchemy.run/gcp"; |
| 110 | * import * as Effect from "effect/Effect"; |
| 111 | * |
| 112 | * export default Alchemy.Stack( |
| 113 | * "MyApp", |
| 114 | * { providers: GCP.providers(), state: GCP.state() }, |
| 115 | * Effect.gen(function* () { |
| 116 | * return {}; |
| 117 | * }), |
| 118 | * ); |
| 119 | * ``` |
| 120 | * |
| 121 | * **Example:** Named store with a key prefix |
| 122 | * ```typescript |
| 123 | * export default Alchemy.Stack( |
| 124 | * "MyApp", |
| 125 | * { |
| 126 | * providers: GCP.providers(), |
| 127 | * state: GCP.state({ name: "platform", prefix: "alchemy" }), |
| 128 | * }, |
| 129 | * Effect.gen(function* () { |
| 130 | * return {}; |
| 131 | * }), |
| 132 | * ); |
| 133 | * ``` |
| 134 | * |
| 135 | * ### Provisioning in CI |
| 136 | * In CI (`CI=true`) a missing bucket fails the run instead of being |
| 137 | * created. Create it once ahead of time: |
| 138 | * |
| 139 | * **Example:** Bootstrap the bucket |
| 140 | * ```sh |
| 141 | * alchemy state bootstrap --backend gcp --profile <your-ci-profile> |
| 142 | * ``` |
| 143 | * |
| 144 | * @resource |
| 145 | */ |
| 146 | export const state = (options: GcsStateOptions = {}) => |
| 147 | Layer.effect( |
| 148 | State, |
| 149 | Effect.gen(function* () { |
| 150 | const scope = yield* Effect.scope; |
| 151 | const context = yield* Effect.context<GcsDeps | GcpEnvironment>(); |
| 152 | |
| 153 | // Bind the lease to this layer's scope. The cached effect runs on the |
| 154 | // first state operation, which may sit in a shorter scope. |
| 155 | const make = Scope.provide(makeGcsState(options), scope).pipe( |
| 156 | recordStateStoreInit, |
| 157 | // Account-level: filed in the global run log, not the project's. |
| 158 | globalLogs, |
| 159 | Effect.orDie, |
| 160 | Effect.provideContext(context), |
| 161 | ); |
| 162 | |
| 163 | return yield* Effect.cached(make); |
| 164 | }), |
| 165 | ).pipe(Layer.provideMerge(stateEnvironment()), Layer.orDie); |
| 166 | |
| 167 | /** |
| 168 | * Construct a Cloud Storage-backed `StateService`. Resolving the project and |
| 169 | * ensuring the bucket run on the first state operation. |
| 170 | */ |
| 171 | export const makeGcsState = (options: GcsStateOptions = {}) => |
| 172 | Effect.gen(function* () { |
| 173 | const context = yield* Effect.context<GcsDeps | GcpEnvironment>(); |
| 174 | |
| 175 | const toError = (cause: unknown) => |
| 176 | new StateStoreError({ |
| 177 | message: |
| 178 | cause instanceof Error |
| 179 | ? cause.message |
| 180 | : typeof cause === "object" && cause !== null && "_tag" in cause |
| 181 | ? `Cloud Storage state store error: ${String(cause._tag)}` |
| 182 | : `Cloud Storage state store error: ${String(cause)}`, |
| 183 | cause: cause instanceof Error ? cause : undefined, |
| 184 | }); |
| 185 | |
| 186 | const run = <A, E>(effect: Effect.Effect<A, E, GcsDeps>): Effect.Effect<A, StateStoreError> => |
| 187 | effect.pipe(Effect.provideContext(context), Effect.mapError(toError)); |
| 188 | |
| 189 | const objects = Effect.gen(function* () { |
| 190 | const { project, region } = yield* GcpEnvironment.current; |
| 191 | if (options.name !== undefined) yield* Provisioning.storeName(options.name); |
| 192 | const bucketName = options.bucketName ?? createStateBucketName(project, options.name); |
| 193 | const ensure = ensureStateBucket(bucketName, project, options.location ?? region); |
| 194 | yield* (yield* stateBucketExists(bucketName)) |
| 195 | ? ensure |
| 196 | : Provisioning.deployMissing( |
| 197 | { |
| 198 | backend: "gcp", |
| 199 | name: options.name, |
| 200 | label: `Cloud Storage state bucket '${bucketName}'`, |
| 201 | configured: options.bucketName !== undefined, |
| 202 | }, |
| 203 | ensure, |
| 204 | ); |
| 205 | const media = yield* makeObjectMedia; |
| 206 | return gcsObjects(bucketName, media, run); |
| 207 | }).pipe(Effect.provideContext(context), Effect.mapError(toError)); |
| 208 | |
| 209 | return yield* makeBucketState({ |
| 210 | id: "gcs", |
| 211 | label: "Cloud Storage", |
| 212 | prefix: options.prefix, |
| 213 | objects, |
| 214 | }); |
| 215 | }); |
| 216 | |
| 217 | type ObjectMedia = Effect.Success<typeof makeObjectMedia>; |
| 218 | |
| 219 | /** The Cloud Storage {@link BucketObjects} of `bucket`, versioned by generation. */ |
| 220 | const gcsObjects = ( |
| 221 | bucket: string, |
| 222 | media: ObjectMedia, |
| 223 | run: <A, E>(effect: Effect.Effect<A, E, GcsDeps>) => Effect.Effect<A, StateStoreError>, |
| 224 | ): BucketObjects => ({ |
| 225 | listKeys: (prefix) => |
| 226 | run( |
| 227 | storage.listObjects.pages({ bucket, prefix }).pipe( |
| 228 | Stream.flatMap((page) => Stream.fromIterable(page.items ?? [])), |
| 229 | Stream.map((object) => object.name), |
| 230 | Stream.filter((name): name is string => name !== undefined), |
| 231 | Stream.runCollect, |
| 232 | Effect.map((names) => Array.from(names)), |
| 233 | ), |
| 234 | ), |
| 235 | listChildren: (prefix) => |
| 236 | run( |
| 237 | storage.listObjects.pages({ bucket, prefix, delimiter: "/" }).pipe( |
| 238 | Stream.flatMap((page) => Stream.fromIterable(page.prefixes ?? [])), |
| 239 | // `{prefix}{name}/` -> `{name}` |
| 240 | Stream.map((p) => p.slice(prefix.length, -1)), |
| 241 | Stream.runCollect, |
| 242 | Effect.map((names) => Array.from(names)), |
| 243 | ), |
| 244 | ), |
| 245 | read: (key) => |
| 246 | run( |
| 247 | media.download({ bucket, object: key }).pipe( |
| 248 | Effect.map((object) => ({ |
| 249 | text: new TextDecoder().decode(object.body), |
| 250 | version: object.generation ?? "", |
| 251 | })), |
| 252 | Effect.catchTag("GCP.Storage.ObjectNotFound", () => Effect.undefined), |
| 253 | ), |
| 254 | ), |
| 255 | write: (key, body, condition) => |
| 256 | run( |
| 257 | media |
| 258 | .upload(bucket, { |
| 259 | name: key, |
| 260 | body, |
| 261 | contentType: "application/json", |
| 262 | ifGenerationMatch: |
| 263 | condition === undefined |
| 264 | ? undefined |
| 265 | : "ifAbsent" in condition |
| 266 | ? "0" |
| 267 | : condition.ifVersion, |
| 268 | }) |
| 269 | .pipe( |
| 270 | Effect.map((object): string | undefined => object.generation ?? ""), |
| 271 | // A lost condition answers 412. |
| 272 | Effect.catchIf( |
| 273 | (error) => error._tag === "GCP.Storage.ObjectRequestFailed" && error.status === 412, |
| 274 | () => Effect.undefined, |
| 275 | ), |
| 276 | ), |
| 277 | ), |
| 278 | delete: (key, ifVersion) => |
| 279 | run( |
| 280 | storage |
| 281 | .deleteObjects({ |
| 282 | bucket, |
| 283 | object: key, |
| 284 | ...(ifVersion === undefined ? {} : { ifGenerationMatch: ifVersion }), |
| 285 | }) |
| 286 | .pipe( |
| 287 | Effect.asVoid, |
| 288 | Effect.catchTag(["NotFound", "ObjectPreconditionFailed"], () => Effect.void), |
| 289 | ), |
| 290 | ), |
| 291 | deleteMany: (keys) => |
| 292 | run( |
| 293 | Effect.forEach( |
| 294 | keys, |
| 295 | (object) => |
| 296 | storage.deleteObjects({ bucket, object }).pipe( |
| 297 | Effect.asVoid, |
| 298 | Effect.catchTag("NotFound", () => Effect.void), |
| 299 | ), |
| 300 | { concurrency: 16, discard: true }, |
| 301 | ), |
| 302 | ), |
| 303 | }); |
| 304 | |
| 305 | /** Settings every state bucket converges to. */ |
| 306 | const desiredIamConfiguration = { |
| 307 | uniformBucketLevelAccess: { enabled: true }, |
| 308 | publicAccessPrevention: "enforced", |
| 309 | }; |
| 310 | |
| 311 | /** The role that lets project owners and editors read and write state. */ |
| 312 | const STATE_OBJECT_ROLE = "roles/storage.objectAdmin"; |
| 313 | |
| 314 | /** |
| 315 | * With uniform bucket-level access, project Owners and Editors only hold |
| 316 | * `storage.legacyBucketOwner`/`legacyBucketWriter`, which can list and |
| 317 | * write objects but not read them. Grant them object access on the state |
| 318 | * bucket so anyone who can deploy to the project can use the store. |
| 319 | */ |
| 320 | const syncStateBucketAccess = (bucket: string, project: string) => |
| 321 | Effect.gen(function* () { |
| 322 | const members = [`projectOwner:${project}`, `projectEditor:${project}`]; |
| 323 | const policy = yield* storage.getIamPolicyBuckets({ bucket }); |
| 324 | const bindings = policy.bindings ?? []; |
| 325 | const existing = bindings.find( |
| 326 | (binding) => binding.role === STATE_OBJECT_ROLE && binding.condition === undefined, |
| 327 | ); |
| 328 | const missing = members.filter((member) => !(existing?.members ?? []).includes(member)); |
| 329 | if (missing.length === 0) return; |
| 330 | yield* storage.setIamPolicyBuckets({ |
| 331 | bucket, |
| 332 | body: { |
| 333 | ...policy, |
| 334 | bindings: existing |
| 335 | ? bindings.map((binding) => |
| 336 | binding === existing |
| 337 | ? { ...binding, members: [...(binding.members ?? []), ...missing] } |
| 338 | : binding, |
| 339 | ) |
| 340 | : [...bindings, { role: STATE_OBJECT_ROLE, members }], |
| 341 | }, |
| 342 | }); |
| 343 | }).pipe( |
| 344 | Effect.retry({ |
| 345 | // A concurrent policy write, or a just-created bucket not yet visible. |
| 346 | while: (error) => |
| 347 | error._tag === "IamPolicyEtagMismatch" || |
| 348 | error._tag === "Conflict" || |
| 349 | error._tag === "NotFound", |
| 350 | schedule: Schedule.spaced("1 second"), |
| 351 | times: 8, |
| 352 | }), |
| 353 | ); |
| 354 | |
| 355 | /** |
| 356 | * Observe-then-ensure the state bucket: create it if missing (tolerating |
| 357 | * create races), then converge uniform access, public access prevention, |
| 358 | * versioning, and object access for project owners and editors. |
| 359 | */ |
| 360 | const ensureStateBucket = (bucket: string, project: string, location: string) => |
| 361 | Effect.gen(function* () { |
| 362 | const observed = yield* storage |
| 363 | .getBuckets({ bucket }) |
| 364 | .pipe(Effect.catchTag("NotFound", () => Effect.undefined)); |
| 365 | if (observed === undefined) { |
| 366 | yield* Effect.logInfo(`Cloud Storage state store: creating bucket ${bucket} in ${location}`); |
| 367 | yield* storage |
| 368 | .insertBuckets({ |
| 369 | project, |
| 370 | body: { |
| 371 | name: bucket, |
| 372 | location, |
| 373 | iamConfiguration: desiredIamConfiguration, |
| 374 | versioning: { enabled: true }, |
| 375 | }, |
| 376 | }) |
| 377 | .pipe( |
| 378 | // A concurrent first use created it first. |
| 379 | Effect.catchTag("Conflict", () => Effect.void), |
| 380 | ); |
| 381 | } else { |
| 382 | const drifted = |
| 383 | observed.iamConfiguration?.uniformBucketLevelAccess?.enabled !== true || |
| 384 | observed.iamConfiguration?.publicAccessPrevention !== "enforced" || |
| 385 | observed.versioning?.enabled !== true; |
| 386 | if (drifted) { |
| 387 | yield* storage.patchBuckets({ |
| 388 | bucket, |
| 389 | body: { |
| 390 | iamConfiguration: desiredIamConfiguration, |
| 391 | versioning: { enabled: true }, |
| 392 | }, |
| 393 | }); |
| 394 | } |
| 395 | } |
| 396 | yield* syncStateBucketAccess(bucket, project); |
| 397 | }).pipe( |
| 398 | Effect.retry({ |
| 399 | while: (error) => error._tag === "Conflict", |
| 400 | schedule: Schedule.max([Schedule.spaced("2 seconds"), Schedule.recurs(5)]), |
| 401 | }), |
| 402 | ); |
| 403 | |
| 404 | /** The profile's GCP environment, resolved the way {@link state} resolves it. */ |
| 405 | export const stateEnvironment = () => |
| 406 | // Fresh per call, so an environment provided to this layer is not |
| 407 | // shadowed by one memoized for another stack in the same run. |
| 408 | Layer.fresh(fromCredentials()).pipe( |
| 409 | Layer.provideMerge(Layer.fresh(GcpCredentials.fromAuthProvider())), |
| 410 | Layer.provideMerge(GcpAuth), |
| 411 | Layer.provideMerge(CredentialsStoreLive), |
| 412 | ); |
| 413 | |
| 414 | /** |
| 415 | * Create the state bucket with its secure defaults, or converge an existing |
| 416 | * one to them: what first use of {@link state} does. |
| 417 | */ |
| 418 | export const provisionStateBucket = ( |
| 419 | bucket: string, |
| 420 | project: string, |
| 421 | location: string, |
| 422 | ): Effect.Effect<void, StateStoreError, GcsDeps> => |
| 423 | ensureStateBucket(bucket, project, location).pipe( |
| 424 | Effect.mapError( |
| 425 | (cause) => |
| 426 | new StateStoreError({ |
| 427 | message: `Could not provision state bucket ${bucket}: ${cause._tag}`, |
| 428 | }), |
| 429 | ), |
| 430 | ); |