@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 430 lines 14.8 KB
Raw
1 import * as crypto from "node:crypto";
2 import type { Credentials } from "@distilled.cloud/gcp/Credentials";
3 import * as storage from "@distilled.cloud/gcp/storage_v1";
4 import { CredentialsStoreLive } from "alchemy/Auth/Credentials";
5 import { type BucketObjects, makeBucketState } from "alchemy/State/BucketState";
6 import * as Provisioning from "alchemy/State/Provisioning";
7 import { State, StateStoreError } from "alchemy/State/State";
8 import { recordStateStoreInit } from "alchemy/Telemetry/Metrics";
9 import { globalLogs } from "alchemy/Util/LogRecord";
10 import * as Effect from "effect/Effect";
11 import type { HttpClient } from "effect/http/HttpClient";
12 import * as Layer from "effect/Layer";
13 import * as Schedule from "effect/Schedule";
14 import * as Scope from "effect/Scope";
15 import * as Stream from "effect/Stream";
16 import { GcpAuth } from "../AuthProvider.ts";
17 import * as GcpCredentials from "../Credentials.ts";
18 import { fromCredentials, GcpEnvironment } from "../Environment.ts";
19 import { makeObjectMedia } from "../Storage/ObjectMedia.ts";
20
21 export interface GcsStateOptions {
22 /**
23 * A store other than the project's default, by logical name: its bucket is
24 * `alchemy-state-<name>-<project>`. The bucket is created on first use,
25 * except in CI, where `alchemy state bootstrap --backend gcp --name <name>`
26 * provisions it ahead.
27 *
28 * @default the project's default store, `alchemy-state-<project>`
29 */
30 name?: string;
31 /**
32 * A full bucket name instead of one derived from {@link GcsStateOptions.name},
33 * e.g. an existing bucket. Created on first use if it does not exist.
34 */
35 bucketName?: string;
36 /**
37 * Key prefix within the bucket under which all state objects are stored,
38 * e.g. `"alchemy"`. A trailing `/` is appended automatically.
39 *
40 * @default "" (bucket root)
41 */
42 prefix?: string;
43 /**
44 * Location of a bucket the store creates, e.g. `"us-central1"` or the
45 * multi-region `"US"`. An existing bucket keeps its location.
46 *
47 * @default the profile's region, else `us-central1`
48 */
49 location?: string;
50 }
51
52 /** Context required by the distilled Cloud Storage operations. */
53 type GcsDeps = Credentials | HttpClient;
54
55 /** Cloud Storage bucket names are at most 63 characters. */
56 const BUCKET_NAME_MAX = 63;
57
58 /**
59 * Build the default state bucket name, `alchemy-state-[<name>-]<project>`.
60 * Bucket names are global, and project ids are unique, so the name is
61 * deterministic per project. A name too long for Cloud Storage replaces the
62 * project id with a hash of it.
63 */
64 export const createStateBucketName = (project: string, name?: string) => {
65 const base = `alchemy-state-${name === undefined ? "" : `${name}-`}`;
66 const full = `${base}${project}`.toLowerCase();
67 if (full.length <= BUCKET_NAME_MAX) return full;
68 const hash = crypto.createHash("sha256").update(project).digest("hex").slice(0, 16);
69 return `${base}${hash}`.toLowerCase();
70 };
71
72 /** Whether a state bucket exists. */
73 export const stateBucketExists = (bucket: string) =>
74 storage.getBuckets({ bucket }).pipe(
75 Effect.as(true),
76 Effect.catchTag("NotFound", () => Effect.succeed(false)),
77 );
78
79 /**
80 * State store backed by a Google Cloud Storage bucket.
81 *
82 * Stack state is persisted as JSON objects in a bucket in your project,
83 * laid out like the local state store's file tree:
84 *
85 * ```
86 * gs://{bucket}/{prefix}{stack}/{stage}/{fqn}.json
87 * gs://{bucket}/{prefix}{stack}/{stage}/__stack_output__.json
88 * gs://{bucket}/{prefix}{stack}/{stage}/__lease__.json
89 * ```
90 *
91 * Concurrent deploys of one stack and stage share an `__lease__.json`
92 * object written with `ifGenerationMatch` preconditions. The second deploy
93 * fails immediately instead of mixing rows.
94 *
95 * The bucket is created on the first state operation if it does not
96 * exist, with uniform bucket-level access, public access prevention, and
97 * object versioning. Project Owners and Editors get
98 * `roles/storage.objectAdmin` on it, since uniform access otherwise leaves
99 * them unable to read objects. Nothing touches GCP credentials at layer
100 * construction time.
101 *
102 * ### Using the Cloud Storage State Store
103 * Pass `GCP.state()` as the `state` option of a Stack. By default the state
104 * is stored in a bucket named `alchemy-state-{project}`.
105 *
106 * **Example:** Default bucket
107 * ```typescript
108 * import * as Alchemy from "alchemy";
109 * import * as GCP from "@alchemy.run/gcp";
110 * import * as Effect from "effect/Effect";
111 *
112 * export default Alchemy.Stack(
113 * "MyApp",
114 * { providers: GCP.providers(), state: GCP.state() },
115 * Effect.gen(function* () {
116 * return {};
117 * }),
118 * );
119 * ```
120 *
121 * **Example:** Named store with a key prefix
122 * ```typescript
123 * export default Alchemy.Stack(
124 * "MyApp",
125 * {
126 * providers: GCP.providers(),
127 * state: GCP.state({ name: "platform", prefix: "alchemy" }),
128 * },
129 * Effect.gen(function* () {
130 * return {};
131 * }),
132 * );
133 * ```
134 *
135 * ### Provisioning in CI
136 * In CI (`CI=true`) a missing bucket fails the run instead of being
137 * created. Create it once ahead of time:
138 *
139 * **Example:** Bootstrap the bucket
140 * ```sh
141 * alchemy state bootstrap --backend gcp --profile <your-ci-profile>
142 * ```
143 *
144 * @resource
145 */
146 export const state = (options: GcsStateOptions = {}) =>
147 Layer.effect(
148 State,
149 Effect.gen(function* () {
150 const scope = yield* Effect.scope;
151 const context = yield* Effect.context<GcsDeps | GcpEnvironment>();
152
153 // Bind the lease to this layer's scope. The cached effect runs on the
154 // first state operation, which may sit in a shorter scope.
155 const make = Scope.provide(makeGcsState(options), scope).pipe(
156 recordStateStoreInit,
157 // Account-level: filed in the global run log, not the project's.
158 globalLogs,
159 Effect.orDie,
160 Effect.provideContext(context),
161 );
162
163 return yield* Effect.cached(make);
164 }),
165 ).pipe(Layer.provideMerge(stateEnvironment()), Layer.orDie);
166
167 /**
168 * Construct a Cloud Storage-backed `StateService`. Resolving the project and
169 * ensuring the bucket run on the first state operation.
170 */
171 export const makeGcsState = (options: GcsStateOptions = {}) =>
172 Effect.gen(function* () {
173 const context = yield* Effect.context<GcsDeps | GcpEnvironment>();
174
175 const toError = (cause: unknown) =>
176 new StateStoreError({
177 message:
178 cause instanceof Error
179 ? cause.message
180 : typeof cause === "object" && cause !== null && "_tag" in cause
181 ? `Cloud Storage state store error: ${String(cause._tag)}`
182 : `Cloud Storage state store error: ${String(cause)}`,
183 cause: cause instanceof Error ? cause : undefined,
184 });
185
186 const run = <A, E>(effect: Effect.Effect<A, E, GcsDeps>): Effect.Effect<A, StateStoreError> =>
187 effect.pipe(Effect.provideContext(context), Effect.mapError(toError));
188
189 const objects = Effect.gen(function* () {
190 const { project, region } = yield* GcpEnvironment.current;
191 if (options.name !== undefined) yield* Provisioning.storeName(options.name);
192 const bucketName = options.bucketName ?? createStateBucketName(project, options.name);
193 const ensure = ensureStateBucket(bucketName, project, options.location ?? region);
194 yield* (yield* stateBucketExists(bucketName))
195 ? ensure
196 : Provisioning.deployMissing(
197 {
198 backend: "gcp",
199 name: options.name,
200 label: `Cloud Storage state bucket '${bucketName}'`,
201 configured: options.bucketName !== undefined,
202 },
203 ensure,
204 );
205 const media = yield* makeObjectMedia;
206 return gcsObjects(bucketName, media, run);
207 }).pipe(Effect.provideContext(context), Effect.mapError(toError));
208
209 return yield* makeBucketState({
210 id: "gcs",
211 label: "Cloud Storage",
212 prefix: options.prefix,
213 objects,
214 });
215 });
216
217 type ObjectMedia = Effect.Success<typeof makeObjectMedia>;
218
219 /** The Cloud Storage {@link BucketObjects} of `bucket`, versioned by generation. */
220 const gcsObjects = (
221 bucket: string,
222 media: ObjectMedia,
223 run: <A, E>(effect: Effect.Effect<A, E, GcsDeps>) => Effect.Effect<A, StateStoreError>,
224 ): BucketObjects => ({
225 listKeys: (prefix) =>
226 run(
227 storage.listObjects.pages({ bucket, prefix }).pipe(
228 Stream.flatMap((page) => Stream.fromIterable(page.items ?? [])),
229 Stream.map((object) => object.name),
230 Stream.filter((name): name is string => name !== undefined),
231 Stream.runCollect,
232 Effect.map((names) => Array.from(names)),
233 ),
234 ),
235 listChildren: (prefix) =>
236 run(
237 storage.listObjects.pages({ bucket, prefix, delimiter: "/" }).pipe(
238 Stream.flatMap((page) => Stream.fromIterable(page.prefixes ?? [])),
239 // `{prefix}{name}/` -> `{name}`
240 Stream.map((p) => p.slice(prefix.length, -1)),
241 Stream.runCollect,
242 Effect.map((names) => Array.from(names)),
243 ),
244 ),
245 read: (key) =>
246 run(
247 media.download({ bucket, object: key }).pipe(
248 Effect.map((object) => ({
249 text: new TextDecoder().decode(object.body),
250 version: object.generation ?? "",
251 })),
252 Effect.catchTag("GCP.Storage.ObjectNotFound", () => Effect.undefined),
253 ),
254 ),
255 write: (key, body, condition) =>
256 run(
257 media
258 .upload(bucket, {
259 name: key,
260 body,
261 contentType: "application/json",
262 ifGenerationMatch:
263 condition === undefined
264 ? undefined
265 : "ifAbsent" in condition
266 ? "0"
267 : condition.ifVersion,
268 })
269 .pipe(
270 Effect.map((object): string | undefined => object.generation ?? ""),
271 // A lost condition answers 412.
272 Effect.catchIf(
273 (error) => error._tag === "GCP.Storage.ObjectRequestFailed" && error.status === 412,
274 () => Effect.undefined,
275 ),
276 ),
277 ),
278 delete: (key, ifVersion) =>
279 run(
280 storage
281 .deleteObjects({
282 bucket,
283 object: key,
284 ...(ifVersion === undefined ? {} : { ifGenerationMatch: ifVersion }),
285 })
286 .pipe(
287 Effect.asVoid,
288 Effect.catchTag(["NotFound", "ObjectPreconditionFailed"], () => Effect.void),
289 ),
290 ),
291 deleteMany: (keys) =>
292 run(
293 Effect.forEach(
294 keys,
295 (object) =>
296 storage.deleteObjects({ bucket, object }).pipe(
297 Effect.asVoid,
298 Effect.catchTag("NotFound", () => Effect.void),
299 ),
300 { concurrency: 16, discard: true },
301 ),
302 ),
303 });
304
305 /** Settings every state bucket converges to. */
306 const desiredIamConfiguration = {
307 uniformBucketLevelAccess: { enabled: true },
308 publicAccessPrevention: "enforced",
309 };
310
311 /** The role that lets project owners and editors read and write state. */
312 const STATE_OBJECT_ROLE = "roles/storage.objectAdmin";
313
314 /**
315 * With uniform bucket-level access, project Owners and Editors only hold
316 * `storage.legacyBucketOwner`/`legacyBucketWriter`, which can list and
317 * write objects but not read them. Grant them object access on the state
318 * bucket so anyone who can deploy to the project can use the store.
319 */
320 const syncStateBucketAccess = (bucket: string, project: string) =>
321 Effect.gen(function* () {
322 const members = [`projectOwner:${project}`, `projectEditor:${project}`];
323 const policy = yield* storage.getIamPolicyBuckets({ bucket });
324 const bindings = policy.bindings ?? [];
325 const existing = bindings.find(
326 (binding) => binding.role === STATE_OBJECT_ROLE && binding.condition === undefined,
327 );
328 const missing = members.filter((member) => !(existing?.members ?? []).includes(member));
329 if (missing.length === 0) return;
330 yield* storage.setIamPolicyBuckets({
331 bucket,
332 body: {
333 ...policy,
334 bindings: existing
335 ? bindings.map((binding) =>
336 binding === existing
337 ? { ...binding, members: [...(binding.members ?? []), ...missing] }
338 : binding,
339 )
340 : [...bindings, { role: STATE_OBJECT_ROLE, members }],
341 },
342 });
343 }).pipe(
344 Effect.retry({
345 // A concurrent policy write, or a just-created bucket not yet visible.
346 while: (error) =>
347 error._tag === "IamPolicyEtagMismatch" ||
348 error._tag === "Conflict" ||
349 error._tag === "NotFound",
350 schedule: Schedule.spaced("1 second"),
351 times: 8,
352 }),
353 );
354
355 /**
356 * Observe-then-ensure the state bucket: create it if missing (tolerating
357 * create races), then converge uniform access, public access prevention,
358 * versioning, and object access for project owners and editors.
359 */
360 const ensureStateBucket = (bucket: string, project: string, location: string) =>
361 Effect.gen(function* () {
362 const observed = yield* storage
363 .getBuckets({ bucket })
364 .pipe(Effect.catchTag("NotFound", () => Effect.undefined));
365 if (observed === undefined) {
366 yield* Effect.logInfo(`Cloud Storage state store: creating bucket ${bucket} in ${location}`);
367 yield* storage
368 .insertBuckets({
369 project,
370 body: {
371 name: bucket,
372 location,
373 iamConfiguration: desiredIamConfiguration,
374 versioning: { enabled: true },
375 },
376 })
377 .pipe(
378 // A concurrent first use created it first.
379 Effect.catchTag("Conflict", () => Effect.void),
380 );
381 } else {
382 const drifted =
383 observed.iamConfiguration?.uniformBucketLevelAccess?.enabled !== true ||
384 observed.iamConfiguration?.publicAccessPrevention !== "enforced" ||
385 observed.versioning?.enabled !== true;
386 if (drifted) {
387 yield* storage.patchBuckets({
388 bucket,
389 body: {
390 iamConfiguration: desiredIamConfiguration,
391 versioning: { enabled: true },
392 },
393 });
394 }
395 }
396 yield* syncStateBucketAccess(bucket, project);
397 }).pipe(
398 Effect.retry({
399 while: (error) => error._tag === "Conflict",
400 schedule: Schedule.max([Schedule.spaced("2 seconds"), Schedule.recurs(5)]),
401 }),
402 );
403
404 /** The profile's GCP environment, resolved the way {@link state} resolves it. */
405 export const stateEnvironment = () =>
406 // Fresh per call, so an environment provided to this layer is not
407 // shadowed by one memoized for another stack in the same run.
408 Layer.fresh(fromCredentials()).pipe(
409 Layer.provideMerge(Layer.fresh(GcpCredentials.fromAuthProvider())),
410 Layer.provideMerge(GcpAuth),
411 Layer.provideMerge(CredentialsStoreLive),
412 );
413
414 /**
415 * Create the state bucket with its secure defaults, or converge an existing
416 * one to them: what first use of {@link state} does.
417 */
418 export const provisionStateBucket = (
419 bucket: string,
420 project: string,
421 location: string,
422 ): Effect.Effect<void, StateStoreError, GcsDeps> =>
423 ensureStateBucket(bucket, project, location).pipe(
424 Effect.mapError(
425 (cause) =>
426 new StateStoreError({
427 message: `Could not provision state bucket ${bucket}: ${cause._tag}`,
428 }),
429 ),
430 );