@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 539 lines 20.3 KB
Raw
1 import type { Credentials } from "@distilled.cloud/aws/Credentials";
2 import type { Region } from "@distilled.cloud/aws/Region";
3 import * as s3 from "@distilled.cloud/aws/s3";
4 import { CredentialsStoreLive } from "alchemy/Auth/Credentials";
5 import { type BucketObjects, makeBucketState } from "alchemy/State/BucketState";
6 import * as Provisioning from "alchemy/State/Provisioning";
7 import { State, StateStoreError } from "alchemy/State/State";
8 import { recordStateStoreInit } from "alchemy/Telemetry/Metrics";
9 import { globalLogs } from "alchemy/Util/LogRecord";
10 import * as Effect from "effect/Effect";
11 import type { HttpClient } from "effect/http/HttpClient";
12 import * as Layer from "effect/Layer";
13 import * as LogLevel from "effect/LogLevel";
14 import * as References from "effect/References";
15 import * as Schedule from "effect/Schedule";
16 import * as Scope from "effect/Scope";
17 import * as Stream from "effect/Stream";
18 import { AwsAuth } from "../AuthProvider.ts";
19 import * as AwsCredentials from "../Credentials.ts";
20 import * as Endpoint from "../Endpoint.ts";
21 import { AWSEnvironment, providedOrDefault } from "../Environment.ts";
22 import * as AwsRegion from "../Region.ts";
23 import { syncBucketEncryption, type BucketEncryption } from "../S3/Bucket.ts";
24
25 /** Maximum number of keys S3 accepts in a single DeleteObjects call. */
26 const DELETE_BATCH_SIZE = 1000;
27
28 export interface S3StateOptions {
29 /**
30 * A store other than the account's default, by logical name: its bucket is
31 * `alchemy-state-<name>-<accountId>-<region>-an`. The bucket is created on
32 * first use, except in CI, where `alchemy state bootstrap --backend aws
33 * --name <name>` provisions it ahead.
34 *
35 * @default the account's default store, `alchemy-state-<accountId>-<region>-an`
36 */
37 name?: string;
38 /**
39 * A full bucket name instead of one derived from {@link S3StateOptions.name},
40 * e.g. an existing bucket. Created on first use like a named store's, in the
41 * account-regional namespace, so it must follow the
42 * `<prefix>-<accountId>-<region>-an` convention.
43 *
44 * @see https://docs.aws.amazon.com/AmazonS3/latest/userguide/gpbucketnamespaces.html#account-regional-gp-buckets
45 */
46 bucketName?: string;
47 /**
48 * Key prefix within the bucket under which all state objects are
49 * stored, e.g. `"alchemy"`. A trailing `/` is appended automatically.
50 *
51 * @default "" (bucket root)
52 */
53 prefix?: string;
54 /**
55 * Default encryption enforced on every fresh state-service initialization.
56 * Omission restores AES256, no KMS key, bucket keys disabled, and no blocked
57 * encryption types. Set `blockedEncryptionTypes: ["SSE-C"]` to block
58 * customer-provided keys. Existing encrypted state objects are not rewritten.
59 *
60 * @default `{ sseAlgorithm: "AES256", bucketKeyEnabled: false, blockedEncryptionTypes: [] }`
61 */
62 encryption?: BucketEncryption;
63 }
64
65 /** Whether a state bucket exists. */
66 export const stateBucketExists = (bucket: string) =>
67 s3.headBucket({ Bucket: bucket }).pipe(
68 Effect.as(true),
69 Effect.catchTag(["NotFound", "NoSuchBucket"], () => Effect.succeed(false)),
70 );
71
72 /** Context required by the distilled S3 operations. */
73 type S3Deps = Credentials | HttpClient | Region;
74
75 /**
76 * Run with Debug and Trace records switched off.
77 *
78 * The AWS client logs every request payload and parsed response at Debug. For
79 * this store those are the serialized state objects, whose values include
80 * secrets Alchemy generated and keeps, so a Debug floor inherited from where
81 * the store is built (the CLI's run log sets one) must not reach them. Floors
82 * already stricter than Info are left alone.
83 */
84 export const withoutSdkDebugLogs = Effect.updateService(References.MinimumLogLevel, (level) =>
85 LogLevel.isLessThan(level, "Info") ? "Info" : level,
86 );
87
88 /**
89 * State store backed by an AWS S3 bucket.
90 *
91 * Stack state is persisted as JSON objects in an account-regional S3
92 * bucket, laid out exactly like the local state store's file tree with
93 * the bucket (plus optional `prefix`) taking the place of the
94 * `.alchemy/state` directory:
95 *
96 * ```
97 * s3://{bucket}/{prefix}{stack}/{stage}/{fqn}.json
98 * s3://{bucket}/{prefix}{stack}/{stage}/__stack_output__.json
99 * s3://{bucket}/{prefix}{stack}/{stage}/__lease__.json
100 * ```
101 *
102 * Concurrent deploys of one stack and stage share an `__lease__.json`
103 * object written with `If-None-Match` / `If-Match`. The second deploy
104 * fails immediately instead of mixing rows. The holder refreshes the
105 * lease until the store closes. A killed process stops refreshing, the
106 * lease expires, and the next deploy can take it.
107 *
108 * The bucket is created lazily on the first state operation if it does
109 * not already exist — nothing touches AWS credentials at layer
110 * construction time.
111 *
112 *
113 * ### Using the S3 State Store
114 * Pass `AWS.state({ backend: "s3" })` as the `state` option of a Stack. By
115 * default the state is stored in an account-regional bucket named
116 * `alchemy-state-{accountId}-{region}-an`.
117 *
118 * **Example:** Default bucket
119 * ```typescript
120 * import * as Alchemy from "alchemy";
121 * import * as AWS from "@alchemy.run/aws";
122 *
123 * const Stack = Alchemy.Stack(
124 * "my-stack",
125 * { providers: AWS.providers(), state: AWS.state({ backend: "s3" }) },
126 * Effect.gen(function* () {
127 * // ...
128 * }),
129 * );
130 * ```
131 *
132 * **Example:** Custom bucket and key prefix
133 * ```typescript
134 * const Stack = Alchemy.Stack(
135 * "my-stack",
136 * {
137 * providers: AWS.providers(),
138 * state: AWS.state({
139 * name: "my-company-state",
140 * prefix: "alchemy",
141 * encryption: {
142 * sseAlgorithm: "aws:kms",
143 * kmsMasterKeyId: "alias/alchemy-state",
144 * },
145 * }),
146 * },
147 * Effect.gen(function* () {
148 * // ...
149 * }),
150 * );
151 * ```
152 *
153 * ### Supplying the AWS Environment
154 * The store resolves its account, region and credentials from the configured
155 * profile, CI credentials or the ambient AWS environment. Provide an
156 * `AWSEnvironment` to use another credential source; provide the same layer
157 * to `AWS.providers()` so the state bucket and every resource share it.
158 *
159 * **Example:** Deploy-role credentials shared with the providers
160 * ```typescript
161 * const Stack = Alchemy.Stack(
162 * "my-stack",
163 * {
164 * providers: AWS.providers().pipe(Layer.provide(environment)),
165 * state: AWS.state({ name: "my-company-state" }).pipe(Layer.provide(environment)),
166 * },
167 * Effect.gen(function* () {
168 * // ...
169 * }),
170 * );
171 * ```
172 *
173 * ### Managing SSE-C Restrictions
174 * **Example:** Block customer-provided encryption keys on the state bucket
175 * ```typescript
176 * const stateStore = AWS.state({
177 * encryption: {
178 * sseAlgorithm: "AES256",
179 * blockedEncryptionTypes: ["SSE-C"],
180 * },
181 * });
182 * ```
183 *
184 * Omitted `blockedEncryptionTypes` is equivalent to `[]`: no encryption types
185 * are blocked, so SSE-C writes are permitted via AWS's `NONE` value. Removing
186 * an explicit block clears it. Omitting `encryption` restores AES256, no KMS
187 * key, disabled bucket keys, and no encryption restrictions. Existing objects
188 * are not rewritten.
189 */
190 export const s3State = (options: S3StateOptions = {}) =>
191 Layer.effect(
192 State,
193 Effect.gen(function* () {
194 const scope = yield* Effect.scope;
195 const context = yield* Effect.context<S3Deps | AWSEnvironment>();
196
197 // Bind the lease to this layer's scope. The cached effect runs on
198 // the first state operation, which may sit in a shorter scope.
199 const make = Scope.provide(makeS3State(options), scope).pipe(
200 recordStateStoreInit,
201 // Account-level: filed in the global run log, not the project's.
202 globalLogs,
203 Effect.orDie,
204 Effect.provideContext(context),
205 );
206
207 return yield* Effect.cached(make);
208 }),
209 ).pipe(Layer.provideMerge(stateEnvironment()), Layer.orDie);
210
211 /**
212 * Construct an S3-backed {@link StateService}.
213 *
214 * Construction itself never touches AWS — environment resolution and
215 * the ensure-bucket-exists check are deferred into a cached Effect
216 * that runs once, on the first state operation.
217 */
218 export const makeS3State = (options: S3StateOptions = {}) =>
219 Effect.gen(function* () {
220 // Captured under `withoutSdkDebugLogs` (below), so every store call runs
221 // with the raised floor.
222 const context = yield* Effect.context<S3Deps | AWSEnvironment>();
223
224 const toError = (cause: unknown) =>
225 new StateStoreError({
226 message: cause instanceof Error ? cause.message : `S3 state store error: ${String(cause)}`,
227 cause: cause instanceof Error ? cause : undefined,
228 });
229
230 // Close over the captured context so every operation is self-contained.
231 const run = <A, E>(effect: Effect.Effect<A, E, S3Deps>): Effect.Effect<A, StateStoreError> =>
232 effect.pipe(Effect.provideContext(context), Effect.mapError(toError));
233
234 // Anything that touches AWS credentials must NOT run at layer
235 // construction time. Resolving the environment (account/region),
236 // deriving the bucket name, and ensuring the bucket exists all run on
237 // the first state operation; `makeBucketState` caches the result.
238 const objects = Effect.gen(function* () {
239 const { accountId, region } = yield* AWSEnvironment.current;
240 if (options.name !== undefined) yield* Provisioning.storeName(options.name);
241 const bucketName =
242 options.bucketName ?? createStateBucketName(accountId, region, options.name);
243 const ensure = ensureStateBucket(bucketName, region, options);
244 // First use creates the bucket (except in CI); every use converges
245 // its settings.
246 yield* (yield* stateBucketExists(bucketName))
247 ? ensure
248 : Provisioning.deployMissing(
249 {
250 backend: "aws-s3",
251 name: options.name,
252 label: `S3 state bucket '${bucketName}'`,
253 configured: options.bucketName !== undefined,
254 },
255 ensure,
256 );
257 return s3Objects(bucketName, run);
258 }).pipe(Effect.provideContext(context), Effect.mapError(toError));
259
260 return yield* makeBucketState({ id: "s3", label: "S3", prefix: options.prefix, objects });
261 }).pipe(withoutSdkDebugLogs);
262
263 /** The S3 {@link BucketObjects} of `bucket`, versioned by ETag. */
264 const s3Objects = (
265 bucket: string,
266 run: <A, E>(effect: Effect.Effect<A, E, S3Deps>) => Effect.Effect<A, StateStoreError>,
267 ): BucketObjects => ({
268 listKeys: (prefix) =>
269 run(
270 s3.listObjectsV2.pages({ Bucket: bucket, Prefix: prefix }).pipe(
271 Stream.flatMap((page) => Stream.fromIterable(page.Contents ?? [])),
272 Stream.map((object) => object.Key),
273 Stream.filter((key): key is string => key !== undefined),
274 Stream.runCollect,
275 Effect.map((keys) => Array.from(keys)),
276 ),
277 ),
278 listChildren: (prefix) =>
279 run(
280 s3.listObjectsV2.pages({ Bucket: bucket, Prefix: prefix, Delimiter: "/" }).pipe(
281 Stream.flatMap((page) => Stream.fromIterable(page.CommonPrefixes ?? [])),
282 // `{prefix}{name}/` -> `{name}`
283 Stream.map((common) => common.Prefix),
284 Stream.filter((p): p is string => p !== undefined),
285 Stream.map((p) => p.slice(prefix.length, -1)),
286 Stream.runCollect,
287 Effect.map((names) => Array.from(names)),
288 ),
289 ),
290 read: (key) =>
291 run(
292 s3.getObject({ Bucket: bucket, Key: key }).pipe(
293 Effect.flatMap((result) =>
294 result.Body === undefined
295 ? Effect.undefined
296 : result.Body.pipe(
297 Stream.decodeText,
298 Stream.mkString,
299 Effect.map((text) => ({ text, version: result.ETag ?? "" })),
300 ),
301 ),
302 Effect.catchTag("NoSuchKey", () => Effect.undefined),
303 ),
304 ),
305 write: (key, body, condition) =>
306 run(
307 s3
308 .putObject({
309 Bucket: bucket,
310 Key: key,
311 Body: body,
312 ContentType: "application/json",
313 ...(condition === undefined
314 ? {}
315 : "ifAbsent" in condition
316 ? { IfNoneMatch: "*" }
317 : { IfMatch: condition.ifVersion }),
318 })
319 .pipe(
320 Effect.flatMap((result) =>
321 result.ETag === undefined
322 ? Effect.fail(
323 new StateStoreError({
324 message: `S3 state object '${key}' was written without an ETag`,
325 }),
326 )
327 : Effect.succeed<string | undefined>(result.ETag),
328 ),
329 Effect.retry({
330 while: (error) => error._tag === "ConditionalRequestConflict",
331 schedule: Schedule.spaced("200 millis"),
332 times: 4,
333 }),
334 // A lost condition: 412, or 409 after retries.
335 Effect.catchTag(
336 ["PreconditionFailed", "ConditionalRequestConflict"],
337 () => Effect.undefined,
338 ),
339 ),
340 ),
341 delete: (key, ifVersion) =>
342 run(
343 s3
344 .deleteObject({
345 Bucket: bucket,
346 Key: key,
347 ...(ifVersion === undefined ? {} : { IfMatch: ifVersion }),
348 })
349 .pipe(Effect.asVoid),
350 ),
351 deleteMany: (keys) =>
352 run(
353 Effect.forEach(
354 Array.from({ length: Math.ceil(keys.length / DELETE_BATCH_SIZE) }, (_, i) =>
355 keys.slice(i * DELETE_BATCH_SIZE, (i + 1) * DELETE_BATCH_SIZE),
356 ),
357 (batch) =>
358 s3.deleteObjects({
359 Bucket: bucket,
360 Delete: { Objects: batch.map((Key) => ({ Key })), Quiet: true },
361 }),
362 { discard: true },
363 ),
364 ),
365 });
366
367 /**
368 * Build the default account-regional state bucket name.
369 *
370 * Account-regional buckets must follow the naming convention:
371 * `<prefix>-<accountId>-<region>-an`
372 *
373 * @see https://docs.aws.amazon.com/AmazonS3/latest/userguide/gpbucketnamespaces.html#account-regional-gp-buckets
374 */
375 export const createStateBucketName = (accountId: string, region: string, name?: string) =>
376 `alchemy-state-${name === undefined ? "" : `${name}-`}${accountId}-${region}-an`.toLowerCase();
377
378 /**
379 * Observe-then-ensure the state bucket: head it, create it if missing
380 * (tolerating create races), and wait for it to become available.
381 */
382 const ensureStateBucket = (bucket: string, region: string, options: S3StateOptions) =>
383 Effect.gen(function* () {
384 // An absent bucket surfaces as either `NotFound` (the HEAD 404) or
385 // `NoSuchBucket` depending on the namespace/path — treat both as "create
386 // it". Catching only `NotFound` let a deleted state bucket (e.g. after a
387 // nuke) escape as an uncaught `NoSuchBucket` instead of being recreated.
388 const exists = yield* s3.headBucket({ Bucket: bucket }).pipe(
389 Effect.map(() => true),
390 Effect.catchTag(["NotFound", "NoSuchBucket"], () => Effect.succeed(false)),
391 );
392 if (!exists) {
393 yield* Effect.logInfo(`S3 state store: creating bucket ${bucket} in ${region}`);
394 yield* s3
395 .createBucket({
396 Bucket: bucket,
397 // account-regional namespace: the bucket name only needs to be
398 // unique within this account+region, so deterministic default
399 // names can't collide with other AWS customers.
400 BucketNamespace: "account-regional",
401 // us-east-1 rejects an explicit LocationConstraint
402 ...(region === "us-east-1"
403 ? {}
404 : {
405 CreateBucketConfiguration: {
406 LocationConstraint: region as s3.BucketLocationConstraint,
407 },
408 }),
409 })
410 .pipe(
411 // Many callers race to create the shared default state bucket on
412 // first use. The loser sees the create already done
413 // (`BucketAlreadyOwnedByYou`/`BucketAlreadyExists`) or mid-flight
414 // (`OperationAborted`). All mean "someone else is creating it".
415 Effect.catchTag(
416 ["BucketAlreadyOwnedByYou", "BucketAlreadyExists", "OperationAborted"],
417 () => Effect.void,
418 ),
419 );
420
421 // Wait for the bucket to become available. Under a concurrent create
422 // the bucket is briefly not yet head-able and object/configuration ops
423 // would race ahead of it.
424 yield* s3.headBucket({ Bucket: bucket }).pipe(
425 Effect.retry({
426 while: (error) => error._tag === "NotFound" || error._tag === "NoSuchBucket",
427 schedule: Schedule.max([Schedule.spaced("1 second"), Schedule.recurs(10)]),
428 }),
429 );
430 }
431
432 // Secure defaults are reconciled for BOTH newly-created and existing
433 // buckets. This makes AWS.state converge out-of-band drift and explicit
434 // custom buckets rather than only securing the greenfield path.
435 const desiredVersioning = "Enabled";
436 const observedVersioning = yield* s3.getBucketVersioning({
437 Bucket: bucket,
438 });
439 if (observedVersioning.Status !== desiredVersioning) {
440 yield* s3.putBucketVersioning({
441 Bucket: bucket,
442 VersioningConfiguration: { Status: desiredVersioning },
443 });
444 }
445
446 yield* syncBucketEncryption(bucket, options.encryption);
447
448 const desiredPublicAccess: s3.PublicAccessBlockConfiguration = {
449 BlockPublicAcls: true,
450 IgnorePublicAcls: true,
451 BlockPublicPolicy: true,
452 RestrictPublicBuckets: true,
453 };
454 const observedPublicAccess = yield* s3.getPublicAccessBlock({ Bucket: bucket }).pipe(
455 Effect.map((result) => result.PublicAccessBlockConfiguration),
456 Effect.catchTag("NoSuchPublicAccessBlockConfiguration", () => Effect.undefined),
457 );
458 const publicAccessFingerprint = (config: s3.PublicAccessBlockConfiguration | undefined) =>
459 JSON.stringify({
460 blockAcls: config?.BlockPublicAcls ?? false,
461 ignoreAcls: config?.IgnorePublicAcls ?? false,
462 blockPolicy: config?.BlockPublicPolicy ?? false,
463 restrictBuckets: config?.RestrictPublicBuckets ?? false,
464 });
465 if (
466 publicAccessFingerprint(observedPublicAccess) !== publicAccessFingerprint(desiredPublicAccess)
467 ) {
468 yield* s3.putPublicAccessBlock({
469 Bucket: bucket,
470 PublicAccessBlockConfiguration: desiredPublicAccess,
471 });
472 }
473
474 const desiredOwnership = "BucketOwnerEnforced";
475 const observedOwnership = yield* s3.getBucketOwnershipControls({ Bucket: bucket }).pipe(
476 Effect.map((result) => result.OwnershipControls?.Rules?.[0]?.ObjectOwnership),
477 Effect.catchTag("OwnershipControlsNotFoundError", () => Effect.undefined),
478 );
479 if (observedOwnership !== desiredOwnership) {
480 yield* s3.putBucketOwnershipControls({
481 Bucket: bucket,
482 OwnershipControls: {
483 Rules: [{ ObjectOwnership: desiredOwnership }],
484 },
485 });
486 }
487 }).pipe(
488 // The whole observe→create→wait sequence races other first-callers of the
489 // shared bucket; `OperationAborted` (conflicting create) and a transiently
490 // absent bucket (`NoSuchBucket`/`NotFound`) are the faces of that race.
491 // Retry the entire sequence so a concurrent create that is still settling
492 // converges instead of surfacing as a `StateStoreError`.
493 Effect.retry({
494 while: (e) =>
495 e._tag === "OperationAborted" || e._tag === "NoSuchBucket" || e._tag === "NotFound",
496 schedule: Schedule.max([Schedule.spaced("2 seconds"), Schedule.recurs(10)]),
497 }),
498 );
499
500 /** The profile's AWS environment, resolved the way {@link state} resolves it. */
501 export const stateEnvironment = () =>
502 // Fresh per call: these derive from the environment below, and shared
503 // (memoized) instances built for another environment in the same run
504 // would shadow an `AWSEnvironment` provided to this layer.
505 Layer.fresh(AwsRegion.fromEnvironment).pipe(
506 Layer.provideMerge(Layer.fresh(AwsCredentials.fromEnvironment)),
507 Layer.provideMerge(Layer.fresh(Endpoint.fromEnvironment)),
508 Layer.provideMerge(providedOrDefault()),
509 Layer.provideMerge(AwsAuth),
510 Layer.provideMerge(CredentialsStoreLive),
511 );
512
513 /**
514 * Create the state bucket with its secure defaults, or converge an existing
515 * one to them: what first use of {@link state} does. With `kmsKeyId`,
516 * objects are encrypted with that KMS key.
517 */
518 export const provisionStateBucket = (
519 bucket: string,
520 region: string,
521 kmsKeyId?: string,
522 ): Effect.Effect<void, StateStoreError, S3Deps> =>
523 ensureStateBucket(
524 bucket,
525 region,
526 kmsKeyId === undefined
527 ? {}
528 : {
529 encryption: { sseAlgorithm: "aws:kms", kmsMasterKeyId: kmsKeyId, bucketKeyEnabled: true },
530 },
531 ).pipe(
532 Effect.mapError(
533 (cause) =>
534 new StateStoreError({
535 message: `Could not provision state bucket ${bucket}: ${cause._tag}`,
536 cause: cause instanceof Error ? cause : undefined,
537 }),
538 ),
539 );