The best way to control Claude Code, Codex and more from any computer or phone

main
ts 1,501 lines 54.2 KB
Raw
1 import { createHash } from "node:crypto";
2 import type { Credentials } from "@distilled.cloud/aws/Credentials";
3 import * as dynamodb from "@distilled.cloud/aws/dynamodb";
4 import type { Region } from "@distilled.cloud/aws/Region";
5 import * as s3 from "@distilled.cloud/aws/s3";
6 import { STATE_STORE_VERSION } from "alchemy/State/HttpStateApi";
7 import * as Provisioning from "alchemy/State/Provisioning";
8 import {
9 type PersistedState,
10 State,
11 type StateService,
12 StateStoreError,
13 } from "alchemy/State/State";
14 import { encodeState, reviveState } from "alchemy/State/StateEncoding";
15 import { recordStateStoreInit } from "alchemy/Telemetry/Metrics";
16 import * as Arr from "effect/Array";
17 import * as Clock from "effect/Clock";
18 import * as Data from "effect/Data";
19 import * as Duration from "effect/Duration";
20 import * as Effect from "effect/Effect";
21 import type { HttpClient } from "effect/http/HttpClient";
22 import * as Layer from "effect/Layer";
23 import * as Schedule from "effect/Schedule";
24 import * as Scope from "effect/Scope";
25 import * as Semaphore from "effect/Semaphore";
26 import * as Stream from "effect/Stream";
27 import { AWSEnvironment } from "../Environment.ts";
28 import {
29 fenceLegacyStage,
30 HELD_PREFIX,
31 hasLegacyStage,
32 type LegacyLocation,
33 listLegacyStacks,
34 listLegacyStages,
35 normalizePrefix,
36 readLegacyOutput,
37 type StageContents,
38 } from "./LegacyLayout.ts";
39 import { type CatalogStatus, migrateToDynamoDB, ownsStage, rollbackToS3 } from "./Migration.ts";
40 import {
41 createStateBucketName,
42 provisionStateBucket,
43 stateBucketExists,
44 stateEnvironment,
45 withoutSdkDebugLogs,
46 } from "./State.ts";
47
48 export interface DynamoDBStateOptions {
49 /**
50 * A store other than the account's default, by logical name: its table is
51 * `alchemy-state-<name>` and its overflow bucket
52 * `alchemy-state-overflow-<name>-<accountId>-<region>-an`. The table is
53 * created on first use, except in CI, where `alchemy state bootstrap
54 * --backend aws --name <name>` provisions it ahead.
55 *
56 * @default the account's default store, table `alchemy-state`
57 */
58 name?: string;
59 /**
60 * A full table name instead of one derived from
61 * {@link DynamoDBStateOptions.name}, e.g. an existing table. It must have
62 * a string partition key `PK` and a string sort key `SK`.
63 */
64 tableName?: string;
65 /**
66 * A full name for the S3 bucket that holds records too large for a
67 * DynamoDB item, instead of one derived from
68 * {@link DynamoDBStateOptions.name}. Created, with the S3 state store's
69 * secure defaults, with the table.
70 *
71 * @default `alchemy-state-overflow-<accountId>-<region>-an`
72 */
73 overflowBucketName?: string;
74 /**
75 * The S3 state store whose stages this store takes over. A stage that is
76 * still there and not yet in DynamoDB is refused until `alchemy state
77 * migrate` moves it; `false` skips the check, and the S3 access it needs.
78 *
79 * @default the S3 store of the same name, `alchemy-state-<name>-<accountId>-<region>-an`
80 */
81 migrateFrom?: { readonly bucketName?: string; readonly prefix?: string } | false;
82 /**
83 * A customer-managed KMS key (id, ARN or alias) that encrypts the table
84 * and the overflow bucket. Applied when the store creates them.
85 *
86 * @default AWS-owned keys
87 */
88 kmsKeyId?: string;
89 /**
90 * Size in bytes of a record's encoded state above which it is stored in
91 * the overflow bucket, with the DynamoDB item holding a reference to it.
92 * A DynamoDB item holds at most 400 KB.
93 *
94 * @default 307200 (300 KiB)
95 */
96 overflowThreshold?: number;
97 }
98
99 /** The backend name `alchemy state ... --backend` addresses this store by. */
100 export const DYNAMODB_STATE_BACKEND = "aws";
101
102 /** Context required by the distilled DynamoDB and S3 operations. */
103 type AwsDeps = Credentials | HttpClient | Region;
104
105 type Item = { [key: string]: dynamodb.AttributeValue | undefined };
106
107 const DEFAULT_OVERFLOW_THRESHOLD = 300 * 1024;
108
109 /** Maximum number of requests DynamoDB accepts in one BatchWriteItem call. */
110 const BATCH_WRITE_SIZE = 25;
111
112 /** Maximum number of keys S3 accepts in one DeleteObjects call. */
113 const S3_DELETE_BATCH_SIZE = 1000;
114
115 /** Backoff for the writes a BatchWriteItem call leaves unprocessed. */
116 const unprocessedBackoff = Schedule.exponential("200 millis").pipe(
117 Schedule.modifyDelay(({ duration }) =>
118 Effect.succeed(Duration.min(duration, Duration.seconds(5))),
119 ),
120 Schedule.upTo({ times: 8 }),
121 );
122
123 /** How long a lease stays valid without a refresh. */
124 const LEASE_TTL_MS = 60_000;
125
126 /** How long a passing lease check is trusted. A failure is never cached. */
127 const LEASE_CHECK_TTL_MS = 5_000;
128
129 /** Sort key prefix of resource and action records. */
130 const RECORD = "RECORD#";
131 /** Sort key of a stage's stack output. */
132 const OUTPUT = "OUTPUT";
133 /** Sort key of a stage's deploy lease. */
134 const LEASE = "LEASE";
135 /** Partition key of the stage catalog. */
136 const CATALOG = "CATALOG";
137
138 /** Escape the key separator out of a stack or stage name. */
139 const part = (name: string) => name.replaceAll("%", "%25").replaceAll("#", "%23");
140
141 const stagePk = (stack: string, stage: string) => `STAGE#${part(stack)}#${part(stage)}`;
142 /** The catalog sort key prefix of a stack's stages. */
143 const catalogPrefix = (stack: string) => `STAGE#${part(stack)}#`;
144 const catalogSk = (stack: string, stage: string) => `${catalogPrefix(stack)}${part(stage)}`;
145
146 const S = (value: string): dynamodb.AttributeValue => ({ S: value });
147 const N = (value: number): dynamodb.AttributeValue => ({ N: String(value) });
148 const str = (item: Item | undefined, name: string) => item?.[name]?.S;
149
150 /**
151 * Build the default table name. Table names are unique per account and
152 * region, so they need no account or region suffix.
153 */
154 export const createStateTableName = (name?: string) =>
155 `alchemy-state${name === undefined ? "" : `-${name}`}`;
156
157 /** S3's limit on the length of a bucket name. */
158 const MAX_BUCKET_NAME = 63;
159
160 /**
161 * Build the default overflow bucket name, in the account-regional namespace.
162 * A store name too long to fit is shortened, with a hash of the full name
163 * keeping it unique.
164 */
165 export const createOverflowBucketName = (accountId: string, region: string, name?: string) => {
166 const suffix = `${accountId}-${region}-an`;
167 const full = `alchemy-state-overflow-${name === undefined ? "" : `${name}-`}${suffix}`;
168 if (name === undefined || full.length <= MAX_BUCKET_NAME) return full.toLowerCase();
169 const hash = createHash("sha256").update(name).digest("hex").slice(0, 8);
170 const room = MAX_BUCKET_NAME - `alchemy-state-overflow--${hash}-${suffix}`.length;
171 return `alchemy-state-overflow-${name.slice(0, Math.max(0, room)).replace(/-+$/, "")}-${hash}-${suffix}`
172 .replace("--", "-")
173 .toLowerCase();
174 };
175
176 const toError = (cause: unknown) =>
177 new StateStoreError({
178 message:
179 cause instanceof Error ? cause.message : `DynamoDB state store error: ${String(cause)}`,
180 cause: cause instanceof Error ? cause : undefined,
181 });
182
183 /**
184 * State store backed by an AWS DynamoDB table.
185 *
186 * Every stack and stage on the account shares one on-demand table with a
187 * string partition key `PK` and sort key `SK`:
188 *
189 * ```
190 * STAGE#{stack}#{stage} RECORD#{sha256(fqn)} one resource's state
191 * STAGE#{stack}#{stage} OUTPUT the stack's output
192 * STAGE#{stack}#{stage} LEASE the deploy lease
193 * CATALOG STAGE#{stack}#{stage} the stage catalog
194 * ```
195 *
196 * A deploy takes the stage's lease, then loads every record of the stage
197 * with one paginated Query and serves reads from that snapshot. Writes go
198 * to DynamoDB immediately, one item each. A record larger than
199 * `overflowThreshold` is stored in an S3 overflow bucket, and its item
200 * holds the object's key.
201 *
202 * Concurrent deploys of one stack and stage contend for the lease, a
203 * conditional put. The second deploy fails immediately instead of mixing
204 * rows. The holder refreshes the lease until the store closes. A killed
205 * process stops refreshing, the lease expires, and the next deploy can
206 * take it.
207 *
208 * The table and overflow bucket are created on the first state operation
209 * if they do not already exist, the table with deletion protection and
210 * point-in-time recovery enabled.
211 *
212 * ### Using the DynamoDB State Store
213 * Pass `AWS.state()` as the `state` option of a Stack.
214 *
215 * **Example:** Default table
216 * ```typescript
217 * import * as Alchemy from "alchemy";
218 * import * as AWS from "@alchemy.run/aws";
219 *
220 * const Stack = Alchemy.Stack(
221 * "my-stack",
222 * { providers: AWS.providers(), state: AWS.state() },
223 * Effect.gen(function* () {
224 * // ...
225 * }),
226 * );
227 * ```
228 *
229 * **Example:** A named store
230 * ```typescript
231 * const Stack = Alchemy.Stack(
232 * "my-stack",
233 * {
234 * providers: AWS.providers(),
235 * state: AWS.state({ name: "team-a" }), // table alchemy-state-team-a
236 * },
237 * Effect.gen(function* () {
238 * // ...
239 * }),
240 * );
241 * ```
242 *
243 * ### Provisioning the Store in CI
244 * In CI a missing table fails the run. Create it, and its overflow bucket,
245 * once ahead of time:
246 *
247 * ```sh
248 * alchemy state bootstrap --backend aws --profile <your-ci-profile>
249 * ```
250 *
251 */
252 export const dynamoDBState = (options: DynamoDBStateOptions = {}) =>
253 Layer.effect(
254 State,
255 Effect.gen(function* () {
256 const scope = yield* Effect.scope;
257 const context = yield* Effect.context<AwsDeps | AWSEnvironment>();
258
259 // Bind the leases to this layer's scope. The cached effect runs on
260 // the first state operation, which may sit in a shorter scope.
261 const make = Scope.provide(makeDynamoDBState(options), scope).pipe(
262 recordStateStoreInit,
263 Effect.orDie,
264 Effect.provideContext(context),
265 );
266
267 return yield* Effect.cached(make);
268 }),
269 ).pipe(Layer.provideMerge(stateEnvironment()), Layer.orDie);
270
271 interface StoredRecord {
272 /** The encoded state, as JSON. */
273 readonly text: string;
274 }
275
276 /** A stage lease this store holds. */
277 export interface HeldLease {
278 readonly token: string;
279 readonly checkLive: Effect.Effect<void, StateStoreError>;
280 }
281
282 /**
283 * A passing check is trusted for `ttlMs`, so a burst of operations does
284 * one round-trip. A failing check is never cached.
285 */
286 const amortizeCheck = (
287 checkLive: Effect.Effect<void, StateStoreError>,
288 ttlMs: number,
289 lastOkAt: number | undefined,
290 ): Effect.Effect<void, StateStoreError> =>
291 Effect.suspend(() => {
292 if (lastOkAt !== undefined && Date.now() - lastOkAt < ttlMs) {
293 return Effect.void;
294 }
295 return checkLive.pipe(
296 Effect.tap(() =>
297 Effect.sync(() => {
298 lastOkAt = Date.now();
299 }),
300 ),
301 );
302 });
303
304 /**
305 * One cached effect per key. A failed effect is dropped rather than cached,
306 * so the next call retries it.
307 */
308 const keyedCache = <A>() => {
309 const mutex = Semaphore.makeUnsafe(1);
310 const entries = new Map<string, Effect.Effect<A, StateStoreError>>();
311 return {
312 entries,
313 get: (key: string, make: Effect.Effect<A, StateStoreError>) =>
314 Semaphore.withPermits(
315 mutex,
316 1,
317 )(
318 Effect.gen(function* () {
319 const existing = entries.get(key);
320 if (existing !== undefined) return existing;
321 const cached = yield* Effect.cached(
322 make.pipe(
323 Effect.tapError(() =>
324 Effect.sync(() => {
325 if (entries.get(key) === cached) entries.delete(key);
326 }),
327 ),
328 ),
329 );
330 entries.set(key, cached);
331 return cached;
332 }),
333 ).pipe(Effect.flatMap((effect) => effect)),
334 };
335 };
336
337 /**
338 * Construct a DynamoDB-backed {@link StateService}. Leases are bound to the
339 * ambient scope; callers that need them to outlive the first operation
340 * bind this effect to a longer scope with `Scope.provide`.
341 *
342 * Construction itself never touches AWS: resolving the environment and
343 * ensuring the table exists run once, on the first state operation.
344 */
345 export const makeDynamoDBState = (options: DynamoDBStateOptions = {}) =>
346 makeStore(options).pipe(withoutSdkDebugLogs);
347
348 const makeStore = Effect.fn(function* (options: DynamoDBStateOptions) {
349 const leaseScope = yield* Effect.scope;
350 // Captured under `withoutSdkDebugLogs` (below), so every store call runs
351 // with the raised floor: request payloads are state, and state holds
352 // secrets.
353 const context = yield* Effect.context<AwsDeps | AWSEnvironment>();
354 const run = <A, E>(effect: Effect.Effect<A, E, AwsDeps>): Effect.Effect<A, StateStoreError> =>
355 effect.pipe(Effect.provideContext(context), Effect.mapError(toError));
356
357 const threshold = options.overflowThreshold ?? DEFAULT_OVERFLOW_THRESHOLD;
358 // Only the stack's config describes a store with its own names or key, so
359 // `--backend aws` can't bootstrap it.
360 const configured =
361 options.tableName !== undefined ||
362 options.overflowBucketName !== undefined ||
363 options.kmsKeyId !== undefined;
364 const label = "DynamoDB";
365
366 // The table, overflow bucket and S3 store this store addresses.
367 const names = yield* Effect.cached(
368 Effect.gen(function* () {
369 const { accountId, region } = yield* AWSEnvironment.current;
370 if (options.name !== undefined) yield* Provisioning.storeName(options.name);
371 const table = options.tableName ?? createStateTableName(options.name);
372 const bucket =
373 options.overflowBucketName ?? createOverflowBucketName(accountId, region, options.name);
374 const legacy: LegacyLocation | undefined =
375 options.migrateFrom === false
376 ? undefined
377 : {
378 bucket:
379 options.migrateFrom?.bucketName ??
380 createStateBucketName(accountId, region, options.name),
381 prefix: normalizePrefix(options.migrateFrom?.prefix),
382 };
383 return { table, bucket, region, legacy };
384 }).pipe(Effect.provideContext(context), Effect.mapError(toError)),
385 );
386
387 const provision = (table: string, bucket: string, region: string) =>
388 Effect.all(
389 [
390 ensureStateTable(table, options.kmsKeyId),
391 provisionStateBucket(bucket, region, options.kmsKeyId),
392 ],
393 { concurrency: 2, discard: true },
394 );
395
396 // Ensure the table and overflow bucket exist on the first operation, so a
397 // missing one fails the run before it changes anything.
398 const target = yield* Effect.cached(
399 Effect.gen(function* () {
400 const { table, bucket, region, legacy } = yield* names;
401 const [observed, bucketExists] = yield* Effect.all(
402 [describeStateTable(table), stateBucketExists(bucket)],
403 { concurrency: 2 },
404 );
405 if (observed === undefined) {
406 yield* Provisioning.deployMissing(
407 {
408 backend: DYNAMODB_STATE_BACKEND,
409 name: options.name,
410 configured,
411 label: `DynamoDB state table '${table}'`,
412 },
413 provision(table, bucket, region),
414 );
415 } else {
416 yield* validateKeySchema(observed);
417 if (observed.TableStatus !== "ACTIVE") yield* waitForTableActive(table);
418 if (!bucketExists) {
419 yield* Provisioning.deployMissing(
420 {
421 backend: DYNAMODB_STATE_BACKEND,
422 name: options.name,
423 configured,
424 label: `DynamoDB state overflow bucket '${bucket}'`,
425 },
426 provisionStateBucket(bucket, region, options.kmsKeyId),
427 );
428 }
429 }
430 return { table, bucket, region, legacy };
431 }).pipe(Effect.provideContext(context), Effect.mapError(toError)),
432 );
433
434 const use = <A, E>(
435 f: (target: {
436 table: string;
437 bucket: string;
438 region: string;
439 legacy: LegacyLocation | undefined;
440 }) => Effect.Effect<A, E, AwsDeps>,
441 ): Effect.Effect<A, StateStoreError> => target.pipe(Effect.flatMap((t) => run(f(t))));
442
443 const hashFqn = (fqn: string) =>
444 Effect.sync(() => createHash("sha256").update(fqn).digest("hex"));
445
446 const recordSk = (fqn: string) => hashFqn(fqn).pipe(Effect.map((hash) => `${RECORD}${hash}`));
447
448 const blobPrefix = (stack: string, stage: string) =>
449 `${encodeURIComponent(stack)}/${encodeURIComponent(stage)}/`;
450
451 /** The encoded text of an item: inline, or read from the overflow bucket. */
452 const itemText = Effect.fn(function* (item: Item) {
453 const inline = str(item, "data");
454 if (inline !== undefined) return inline;
455 const ref = str(item, "ref");
456 if (ref === undefined) {
457 return yield* new StateStoreError({
458 message: `DynamoDB state item ${str(item, "PK")}/${str(item, "SK")} has neither data nor ref`,
459 });
460 }
461 const text = yield* use(({ bucket }) =>
462 s3.getObject({ Bucket: bucket, Key: ref }).pipe(
463 Effect.flatMap((result) =>
464 result.Body === undefined
465 ? Effect.void
466 : result.Body.pipe(Stream.decodeText, Stream.mkString),
467 ),
468 Effect.catchTag("NoSuchKey", () => Effect.void),
469 ),
470 );
471 // A missing body is corruption, never a missing resource.
472 if (text === undefined) {
473 return yield* new StateStoreError({
474 message: `DynamoDB state item ${str(item, "PK")}/${str(item, "SK")} references missing overflow object '${ref}'`,
475 });
476 }
477 return text;
478 });
479
480 const parse = <T>(text: string, what: string) =>
481 Effect.try({
482 try: () => JSON.parse(text, reviveState) as T,
483 catch: (cause) =>
484 new StateStoreError({
485 message: `Failed to parse DynamoDB state ${what}`,
486 cause: cause instanceof Error ? cause : undefined,
487 }),
488 });
489
490 /**
491 * Write an item holding `value`, inline or as an overflow reference.
492 * The overflow object is uploaded before the item that references it is
493 * committed, and the object the item referenced before is deleted after.
494 */
495 const putPayload = Effect.fn(function* (
496 key: { PK: string; SK: string; stack: string; stage: string },
497 text: string,
498 attributes: Item,
499 ) {
500 const bytes = yield* Effect.sync(() => new TextEncoder().encode(text).length);
501 let payload: Item;
502 if (bytes > threshold) {
503 // A fresh key per write, so the committed item's object is never
504 // overwritten in place.
505 const ref = `${blobPrefix(key.stack, key.stage)}${encodeURIComponent(key.SK)}-${yield* Effect.sync(() => crypto.randomUUID())}.json`;
506 yield* use(({ bucket }) =>
507 s3.putObject({
508 Bucket: bucket,
509 Key: ref,
510 Body: text,
511 ContentType: "application/json",
512 }),
513 );
514 payload = { ref: S(ref) };
515 } else {
516 payload = { data: S(text) };
517 }
518 const previous = yield* use(({ table }) =>
519 dynamodb.putItem({
520 TableName: table,
521 Item: { PK: S(key.PK), SK: S(key.SK), ...attributes, ...payload },
522 ReturnValues: "ALL_OLD",
523 }),
524 );
525 yield* deleteOverflow(str(previous.Attributes, "ref"), str(payload, "ref"));
526 });
527
528 /** Delete an overflow object no item references any more. Best effort. */
529 const deleteOverflow = (ref: string | undefined, keep?: string) =>
530 ref === undefined || ref === keep
531 ? Effect.void
532 : use(({ bucket }) => s3.deleteObject({ Bucket: bucket, Key: ref })).pipe(Effect.ignore);
533
534 /** Delete items in batches, retrying the ones DynamoDB leaves unprocessed. */
535 const deleteItems = (keys: ReadonlyArray<{ PK: string; SK: string }>) =>
536 use(({ table }) =>
537 Effect.forEach(
538 Arr.chunksOf(keys, BATCH_WRITE_SIZE),
539 (batch) =>
540 Effect.gen(function* () {
541 let requests: dynamodb.WriteRequest[] = batch.map((key) => ({
542 DeleteRequest: { Key: { PK: S(key.PK), SK: S(key.SK) } },
543 }));
544 yield* Effect.gen(function* () {
545 const result = yield* dynamodb.batchWriteItem({
546 RequestItems: { [table]: requests },
547 });
548 requests = result.UnprocessedItems?.[table] ?? [];
549 }).pipe(
550 Effect.repeat({
551 schedule: unprocessedBackoff,
552 until: () => requests.length === 0,
553 }),
554 );
555 if (requests.length > 0) {
556 return yield* new StateStoreError({
557 message: `DynamoDB left ${requests.length} state deletes unprocessed`,
558 });
559 }
560 }),
561 { discard: true },
562 ),
563 );
564
565 /** Every item of a partition, across pagination, strongly consistent. */
566 const queryPartition = (pk: string, skPrefix?: string, projection?: string) =>
567 use(({ table }) =>
568 dynamodb.query
569 .pages({
570 TableName: table,
571 ConsistentRead: true,
572 KeyConditionExpression:
573 skPrefix === undefined ? "PK = :pk" : "PK = :pk AND begins_with(SK, :sk)",
574 ExpressionAttributeValues:
575 skPrefix === undefined ? { ":pk": S(pk) } : { ":pk": S(pk), ":sk": S(skPrefix) },
576 ...(projection === undefined
577 ? {}
578 : {
579 ProjectionExpression: projection,
580 ExpressionAttributeNames: { "#ref": "ref" },
581 }),
582 })
583 .pipe(
584 Stream.flatMap((page) => Stream.fromIterable(page.Items ?? [])),
585 Stream.runCollect,
586 Effect.map((items) => Array.from(items)),
587 ),
588 );
589
590 const heldError = (stack: string, stage: string) =>
591 new StateStoreError({
592 message: `${HELD_PREFIX} the ${label} state lock '${stack}/${stage}'`,
593 });
594
595 const lostError = (stack: string, stage: string) =>
596 new StateStoreError({
597 message: `the ${label} state lock '${stack}/${stage}' was lost mid-run; refusing to continue unlocked`,
598 });
599
600 const leaseKey = (stack: string, stage: string) => ({
601 PK: S(stagePk(stack, stage)),
602 SK: S(LEASE),
603 });
604
605 const catalogKey = (stack: string, stage: string) => ({
606 PK: S(CATALOG),
607 SK: S(catalogSk(stack, stage)),
608 });
609
610 /** Write the lease when `condition` holds; `false` when it doesn't. */
611 const putLease = (
612 stack: string,
613 stage: string,
614 token: string,
615 expiresAt: number,
616 condition: {
617 ConditionExpression: string;
618 ExpressionAttributeNames: Record<string, string>;
619 ExpressionAttributeValues: Item;
620 },
621 ) =>
622 use(({ table }) =>
623 dynamodb
624 .putItem({
625 TableName: table,
626 Item: { ...leaseKey(stack, stage), token: S(token), expiresAt: N(expiresAt) },
627 ...condition,
628 })
629 .pipe(
630 Effect.as(true),
631 Effect.catchTag("ConditionalCheckFailedException", () => Effect.succeed(false)),
632 ),
633 );
634
635 /** Take the stage's lease, held until `scope` closes. */
636 const acquireLease = Effect.fn(function* (stack: string, stage: string, scope: Scope.Scope) {
637 const token = yield* Effect.sync(() => crypto.randomUUID());
638 const now = yield* Clock.currentTimeMillis;
639 // Take the lease when nobody holds it or the holder let it expire.
640 const acquired = yield* putLease(stack, stage, token, now + LEASE_TTL_MS, {
641 ConditionExpression: "attribute_not_exists(PK) OR #expiresAt <= :now",
642 ExpressionAttributeNames: { "#expiresAt": "expiresAt" },
643 ExpressionAttributeValues: { ":now": N(now) },
644 });
645 if (!acquired) return yield* heldError(stack, stage);
646
647 const cursor = { lost: false };
648 const gate = Semaphore.makeUnsafe(1);
649
650 const renew = Semaphore.withPermits(
651 gate,
652 1,
653 )(
654 Effect.gen(function* () {
655 if (cursor.lost) return;
656 const renewedAt = yield* Clock.currentTimeMillis;
657 const renewed = yield* putLease(stack, stage, token, renewedAt + LEASE_TTL_MS, {
658 ConditionExpression: "#token = :token",
659 ExpressionAttributeNames: { "#token": "token" },
660 ExpressionAttributeValues: { ":token": S(token) },
661 });
662 if (!renewed) cursor.lost = true;
663 }),
664 ).pipe(
665 Effect.retry({
666 while: () => !cursor.lost,
667 schedule: Schedule.spaced("1 second"),
668 times: 3,
669 }),
670 Effect.catch(() =>
671 Effect.sync(() => {
672 cursor.lost = true;
673 }),
674 ),
675 );
676
677 const verify = Semaphore.withPermits(
678 gate,
679 1,
680 )(
681 Effect.gen(function* () {
682 if (cursor.lost) return yield* lostError(stack, stage);
683 const existing = yield* use(({ table }) =>
684 dynamodb.getItem({
685 TableName: table,
686 Key: leaseKey(stack, stage),
687 ConsistentRead: true,
688 }),
689 );
690 if (str(existing.Item, "token") !== token) {
691 cursor.lost = true;
692 return yield* lostError(stack, stage);
693 }
694 const observedAt = yield* Clock.currentTimeMillis;
695 if (Number(existing.Item?.expiresAt?.N ?? 0) <= observedAt) {
696 // The lease lapsed without anyone taking it: a refresh queued
697 // behind a large deploy's writes missed its window. Take it back,
698 // on condition it is still ours, so a deploy that took it in the
699 // meantime wins and this one stops.
700 const reclaimed = yield* putLease(stack, stage, token, observedAt + LEASE_TTL_MS, {
701 ConditionExpression: "#token = :token",
702 ExpressionAttributeNames: { "#token": "token" },
703 ExpressionAttributeValues: { ":token": S(token) },
704 });
705 if (!reclaimed) {
706 cursor.lost = true;
707 return yield* lostError(stack, stage);
708 }
709 }
710 }),
711 );
712
713 // Stop the refresh before releasing, so a refresh cannot land after
714 // the lease has been released.
715 yield* Scope.addFinalizer(
716 scope,
717 Semaphore.withPermits(
718 gate,
719 1,
720 )(
721 Effect.gen(function* () {
722 cursor.lost = true;
723 yield* use(({ table }) =>
724 dynamodb
725 .deleteItem({
726 TableName: table,
727 Key: leaseKey(stack, stage),
728 ConditionExpression: "#token = :token",
729 ExpressionAttributeNames: { "#token": "token" },
730 ExpressionAttributeValues: { ":token": S(token) },
731 })
732 .pipe(Effect.catchTag("ConditionalCheckFailedException", () => Effect.void)),
733 );
734 }).pipe(Effect.ignore),
735 ),
736 );
737 yield* renew.pipe(
738 Effect.delay(Duration.millis(Math.floor(LEASE_TTL_MS / 3))),
739 Effect.forever,
740 Effect.interruptible,
741 Effect.forkIn(scope),
742 );
743
744 // The conditional put just proved the lease, so the first check is free.
745 return {
746 token,
747 checkLive: amortizeCheck(verify, LEASE_CHECK_TTL_MS, Date.now()),
748 } satisfies HeldLease;
749 });
750
751 /**
752 * Write one catalog item in a transaction with a check that `lease` is
753 * still the stage's. A holder that stalled past its lease, and lost it to
754 * another deploy, can't move the stage in or out of the catalog.
755 */
756 const writeUnderLease = (
757 stack: string,
758 stage: string,
759 lease: HeldLease,
760 write: (table: string) => dynamodb.TransactWriteItem,
761 ) =>
762 use(({ table }) =>
763 dynamodb
764 .transactWriteItems({
765 TransactItems: [
766 {
767 ConditionCheck: {
768 TableName: table,
769 Key: leaseKey(stack, stage),
770 ConditionExpression: "#token = :token",
771 ExpressionAttributeNames: { "#token": "token" },
772 ExpressionAttributeValues: { ":token": S(lease.token) },
773 },
774 },
775 write(table),
776 ],
777 })
778 .pipe(
779 // A refresh of the lease landed during the transaction.
780 Effect.retry({
781 while: (error) =>
782 error._tag === "TransactionCanceledException" &&
783 cancellationCodes(error).includes("TransactionConflict"),
784 schedule: Schedule.exponential("100 millis"),
785 times: 5,
786 }),
787 Effect.catchTag("TransactionCanceledException", (error) =>
788 Effect.fail(
789 cancellationCodes(error)[0] === "ConditionalCheckFailed"
790 ? lostError(stack, stage)
791 : error,
792 ),
793 ),
794 ),
795 ).pipe(Effect.asVoid);
796
797 const migrateCommand = (stack: string, stage: string) =>
798 `alchemy state migrate --stack ${stack} --stage ${stage}`;
799
800 /** The stage's catalog status; `undefined` when DynamoDB doesn't know it. */
801 const catalogStatus = (stack: string, stage: string) =>
802 use(({ table }) =>
803 dynamodb.getItem({
804 TableName: table,
805 Key: catalogKey(stack, stage),
806 ConsistentRead: true,
807 }),
808 ).pipe(Effect.map((result) => statusOf(result.Item)));
809
810 /**
811 * Refuse a stage the S3 store still has and DynamoDB doesn't own: its
812 * state has not moved, and serving an empty stage would redeploy it.
813 */
814 const refuseUnmigrated = Effect.fn(function* (stack: string, stage: string) {
815 const status = yield* catalogStatus(stack, stage);
816 const { legacy } = yield* names;
817 if (legacy === undefined || ownsStage(status)) return status;
818 if (!(yield* use(() => hasLegacyStage(legacy, stack, stage)))) return status;
819 const command = migrateCommand(stack, stage);
820 return yield* new StateStoreError({
821 message: `Stage '${stack}/${stage}' is still in the S3 state store (s3://${legacy.bucket}/${legacy.prefix}). Run '${command}' to move it to DynamoDB.`,
822 requires: { action: "upgrade", command },
823 });
824 });
825
826 /** Every item of a stage as contents, without the migration check. */
827 const readStage = Effect.fn(function* (stack: string, stage: string) {
828 const items = yield* queryPartition(stagePk(stack, stage));
829 const records = new Map<string, string>();
830 let output: string | undefined;
831 yield* Effect.forEach(
832 items,
833 (item) =>
834 Effect.gen(function* () {
835 const sk = str(item, "SK");
836 if (sk === OUTPUT) output = yield* itemText(item);
837 else if (sk?.startsWith(RECORD)) {
838 const fqn = str(item, "fqn");
839 if (fqn !== undefined) records.set(fqn, yield* itemText(item));
840 }
841 }),
842 { concurrency: 16, discard: true },
843 );
844 return { records, output } satisfies StageContents;
845 });
846
847 /** Load every record of a stage into a snapshot keyed by FQN. */
848 const loadStage = Effect.fn(function* (stack: string, stage: string) {
849 const [items, status] = yield* Effect.all(
850 [queryPartition(stagePk(stack, stage), RECORD), refuseUnmigrated(stack, stage)],
851 { concurrency: 2 },
852 );
853 const records = yield* Effect.forEach(
854 items,
855 (item) =>
856 Effect.gen(function* () {
857 const fqn = str(item, "fqn");
858 if (fqn === undefined) {
859 return yield* new StateStoreError({
860 message: `DynamoDB state item ${str(item, "PK")}/${str(item, "SK")} has no fqn`,
861 });
862 }
863 return [fqn, { text: yield* itemText(item) }] as const;
864 }),
865 { concurrency: 16 },
866 );
867 return { records: new Map<string, StoredRecord>(records), status } satisfies Snapshot;
868 });
869
870 // One lease per (stack, stage), taken on the first operation and held
871 // for the store's lifetime. The snapshot is cached apart from the lease
872 // so a failed load retries without re-taking the lease this store holds.
873 const leases = keyedCache<HeldLease>();
874 const snapshots = keyedCache<Snapshot>();
875 // Stages this store has entered in the catalog.
876 const registered = keyedCache<void>();
877
878 const guarded = <A>(
879 request: { stack: string; stage: string },
880 op: (snapshot: Snapshot, lease: HeldLease) => Effect.Effect<A, StateStoreError>,
881 ): Effect.Effect<A, StateStoreError> => {
882 const key = `${request.stack}/${request.stage}`;
883 return leases.get(key, acquireLease(request.stack, request.stage, leaseScope)).pipe(
884 Effect.tap((lease) => lease.checkLive),
885 Effect.flatMap((lease) =>
886 snapshots
887 .get(key, loadStage(request.stack, request.stage))
888 .pipe(Effect.flatMap((snapshot) => op(snapshot, lease))),
889 ),
890 );
891 };
892
893 const verifyHeldLeases: Effect.Effect<void, StateStoreError> = Effect.suspend(() =>
894 Effect.forEach(
895 Array.from(leases.entries.values()),
896 (lease) => lease.pipe(Effect.flatMap((held) => held.checkLive)),
897 { discard: true },
898 ),
899 );
900
901 /**
902 * Enter the stage in the catalog `listStacks` and `listStages` read, once
903 * per stage per store.
904 */
905 const register = (stack: string, stage: string, snapshot: Snapshot, lease: HeldLease) =>
906 registered.get(
907 `${stack}/${stage}`,
908 Effect.gen(function* () {
909 const { legacy, table } = yield* names;
910 // A stage new to DynamoDB is fenced in the S3 store too, so a
911 // teammate still on the S3 store fails at its lock instead of
912 // deploying the stage a second time.
913 if (
914 legacy !== undefined &&
915 (snapshot.status === undefined || snapshot.status === "rolled-back")
916 ) {
917 const fence = yield* use(() => fenceLegacyStage(legacy, stack, stage, table));
918 if (!fence.fenced) {
919 return yield* new StateStoreError({
920 message: `${HELD_PREFIX} the S3 state lock '${stack}/${stage}': a deploy on the S3 state store is creating this stage`,
921 });
922 }
923 }
924 // Keep the entry's migration record; a destroyed stage is live again.
925 yield* writeUnderLease(stack, stage, lease, (table) => ({
926 Update: {
927 TableName: table,
928 Key: catalogKey(stack, stage),
929 UpdateExpression: "SET #stack = :stack, #stage = :stage REMOVE #status",
930 ExpressionAttributeNames: { "#stack": "stack", "#stage": "stage", "#status": "status" },
931 ExpressionAttributeValues: { ":stack": S(stack), ":stage": S(stage) },
932 },
933 }));
934 snapshot.status = "active";
935 }),
936 );
937
938 /** Every cataloged stage, of one stack or all of them, in one query. */
939 const catalog = (stack?: string) =>
940 queryPartition(CATALOG, stack === undefined ? "STAGE#" : catalogPrefix(stack)).pipe(
941 Effect.map((items) =>
942 items.flatMap((item) => {
943 const stack = str(item, "stack");
944 const stage = str(item, "stage");
945 const status = statusOf(item);
946 return stack === undefined || stage === undefined || status === undefined
947 ? []
948 : [{ stack, stage, status }];
949 }),
950 ),
951 );
952
953 const stageKey = (stack: string, stage: string) => `${stack}\u0000${stage}`;
954
955 /** Stages of a stack, in DynamoDB or still in the S3 store. */
956 const listStages = (stack: string) =>
957 Effect.all(
958 [
959 catalog(stack),
960 names.pipe(
961 Effect.flatMap(({ legacy }) =>
962 legacy === undefined ? Effect.succeed([]) : use(() => listLegacyStages(legacy, stack)),
963 ),
964 ),
965 ],
966 { concurrency: 2 },
967 ).pipe(
968 Effect.map(([entries, legacy]) => {
969 const owned = new Set(entries.filter((e) => ownsStage(e.status)).map((e) => e.stage));
970 return Array.from(
971 new Set([
972 ...entries.filter((entry) => entry.status === "active").map((entry) => entry.stage),
973 ...legacy.filter((stage) => !owned.has(stage)),
974 ]),
975 );
976 }),
977 );
978
979 /** Delete a stage's records, output and overflow objects. */
980 const deleteStageItems = Effect.fn(function* (stack: string, stage: string) {
981 const items = (yield* queryPartition(stagePk(stack, stage), undefined, "PK, SK, #ref")).filter(
982 // Leave the lease. Releasing it here would open the stage before
983 // this delete finishes, and the store still holds the lock.
984 (item) => str(item, "SK") !== LEASE,
985 );
986 yield* deleteItems(items.map((item) => ({ PK: str(item, "PK")!, SK: str(item, "SK")! })));
987 // Overflow objects of the stage, including any a crashed write
988 // uploaded but never committed.
989 yield* use(({ bucket }) =>
990 Effect.gen(function* () {
991 if (!(yield* stateBucketExists(bucket))) return;
992 const objects = yield* s3.listObjectsV2
993 .pages({ Bucket: bucket, Prefix: blobPrefix(stack, stage) })
994 .pipe(
995 Stream.flatMap((page) => Stream.fromIterable(page.Contents ?? [])),
996 Stream.map((object) => object.Key),
997 Stream.filter((key): key is string => key !== undefined),
998 Stream.runCollect,
999 Effect.map((keys) => Array.from(keys)),
1000 );
1001 yield* Effect.forEach(
1002 Arr.chunksOf(objects, S3_DELETE_BATCH_SIZE),
1003 (batch) =>
1004 s3.deleteObjects({
1005 Bucket: bucket,
1006 Delete: { Objects: batch.map((Key) => ({ Key })), Quiet: true },
1007 }),
1008 { discard: true },
1009 );
1010 }),
1011 );
1012 });
1013
1014 /** Delete a stage. Its catalog entry stays as a tombstone while an S3 copy may exist. */
1015 const deleteStage = (stack: string, stage: string) =>
1016 guarded({ stack, stage }, (snapshot, lease) =>
1017 Effect.gen(function* () {
1018 yield* deleteStageItems(stack, stage);
1019 snapshot.records.clear();
1020 if (snapshot.status !== undefined) {
1021 const { legacy } = yield* names;
1022 // With an S3 store behind it, a deleted entry would read as a
1023 // stage that never moved, and bring its pre-migration copy back.
1024 yield* writeUnderLease(stack, stage, lease, (table) =>
1025 legacy === undefined
1026 ? { Delete: { TableName: table, Key: catalogKey(stack, stage) } }
1027 : {
1028 Update: {
1029 TableName: table,
1030 Key: catalogKey(stack, stage),
1031 UpdateExpression: "SET #stack = :stack, #stage = :stage, #status = :destroyed",
1032 ExpressionAttributeNames: {
1033 "#stack": "stack",
1034 "#stage": "stage",
1035 "#status": "status",
1036 },
1037 ExpressionAttributeValues: {
1038 ":stack": S(stack),
1039 ":stage": S(stage),
1040 ":destroyed": S("destroyed"),
1041 },
1042 },
1043 },
1044 );
1045 snapshot.status = legacy === undefined ? undefined : "destroyed";
1046 }
1047 // A later write to the stage enters it in the catalog again.
1048 registered.entries.delete(`${stack}/${stage}`);
1049 }),
1050 );
1051
1052 const state: StateService = {
1053 id: "dynamodb",
1054 getVersion: () => Effect.succeed(STATE_STORE_VERSION),
1055 // A stack is listed while it has a live stage in DynamoDB, or a stage
1056 // the S3 store still has.
1057 listStacks: () =>
1058 verifyHeldLeases.pipe(
1059 Effect.andThen(Effect.all([catalog(), names], { concurrency: 2 })),
1060 Effect.flatMap(([entries, { legacy }]) => {
1061 const owned = new Set(
1062 entries.filter((e) => ownsStage(e.status)).map((e) => stageKey(e.stack, e.stage)),
1063 );
1064 const live = entries.filter((entry) => entry.status === "active").map((e) => e.stack);
1065 if (legacy === undefined) return Effect.succeed(Array.from(new Set(live)));
1066 return use(() =>
1067 listLegacyStacks(legacy).pipe(
1068 Effect.flatMap((stacks) =>
1069 Effect.forEach(
1070 stacks,
1071 (stack) =>
1072 listLegacyStages(legacy, stack).pipe(
1073 Effect.map((stages) =>
1074 stages.some((stage) => !owned.has(stageKey(stack, stage))) ? [stack] : [],
1075 ),
1076 ),
1077 { concurrency: 8 },
1078 ),
1079 ),
1080 ),
1081 ).pipe(
1082 Effect.map((legacyStacks) => Array.from(new Set([...live, ...legacyStacks.flat()]))),
1083 );
1084 }),
1085 ),
1086 listStages: (stack) => verifyHeldLeases.pipe(Effect.andThen(listStages(stack))),
1087 get: (request) =>
1088 guarded(request, ({ records }) => {
1089 const record = records.get(request.fqn);
1090 return record === undefined
1091 ? Effect.undefined
1092 : parse<PersistedState>(record.text, `record '${request.fqn}'`);
1093 }),
1094 getReplacedResources: (request) =>
1095 guarded(request, ({ records }) =>
1096 Effect.forEach(Array.from(records), ([fqn, record]) =>
1097 parse<PersistedState>(record.text, `record '${fqn}'`),
1098 ).pipe(Effect.map((values) => values.filter((value) => value.status === "replaced"))),
1099 ) as ReturnType<StateService["getReplacedResources"]>,
1100 set: (request) =>
1101 guarded(request, (snapshot, lease) =>
1102 Effect.gen(function* () {
1103 yield* register(request.stack, request.stage, snapshot, lease);
1104 const text = JSON.stringify(encodeState(request.value));
1105 yield* putPayload(
1106 {
1107 PK: stagePk(request.stack, request.stage),
1108 SK: yield* recordSk(request.fqn),
1109 stack: request.stack,
1110 stage: request.stage,
1111 },
1112 text,
1113 { fqn: S(request.fqn) },
1114 );
1115 snapshot.records.set(request.fqn, { text });
1116 return request.value;
1117 }),
1118 ),
1119 delete: (request) =>
1120 guarded(request, ({ records }) =>
1121 Effect.gen(function* () {
1122 const previous = yield* use(({ table }) =>
1123 Effect.flatMap(recordSk(request.fqn), (sk) =>
1124 dynamodb.deleteItem({
1125 TableName: table,
1126 Key: { PK: S(stagePk(request.stack, request.stage)), SK: S(sk) },
1127 ReturnValues: "ALL_OLD",
1128 }),
1129 ),
1130 );
1131 records.delete(request.fqn);
1132 yield* deleteOverflow(str(previous.Attributes, "ref"));
1133 }),
1134 ),
1135 // A stack-wide delete locks every stage it is about to remove, so a
1136 // concurrent deploy fails the lease instead of racing the delete.
1137 deleteStack: ({ stack, stage }) =>
1138 stage === undefined
1139 ? verifyHeldLeases.pipe(
1140 Effect.andThen(listStages(stack)),
1141 Effect.flatMap((stages) =>
1142 Effect.forEach(stages, (found) => deleteStage(stack, found), { discard: true }),
1143 ),
1144 )
1145 : deleteStage(stack, stage),
1146 list: (request) =>
1147 guarded(request, ({ records }) => Effect.succeed(Array.from(records.keys()))),
1148 // Always read fresh, without the lease: cross-stack references read
1149 // the outputs of stages another deploy may hold.
1150 getOutput: Effect.fn(function* (request) {
1151 const result = yield* use(({ table }) =>
1152 dynamodb.getItem({
1153 TableName: table,
1154 Key: { PK: S(stagePk(request.stack, request.stage)), SK: S(OUTPUT) },
1155 ConsistentRead: true,
1156 }),
1157 );
1158 if (result.Item !== undefined) {
1159 return yield* parse<unknown>(yield* itemText(result.Item), "stack output");
1160 }
1161 // A stage that hasn't moved yet keeps its output in the S3 store.
1162 // Reading it takes no lease, so a reference to it still resolves.
1163 const { legacy } = yield* names;
1164 if (legacy === undefined) return undefined;
1165 if (ownsStage(yield* catalogStatus(request.stack, request.stage))) return undefined;
1166 return yield* use(() => readLegacyOutput(legacy, request.stack, request.stage));
1167 }),
1168 setOutput: (request) =>
1169 guarded(request, (snapshot, lease) =>
1170 Effect.gen(function* () {
1171 if (request.value === undefined) {
1172 const previous = yield* use(({ table }) =>
1173 dynamodb.deleteItem({
1174 TableName: table,
1175 Key: { PK: S(stagePk(request.stack, request.stage)), SK: S(OUTPUT) },
1176 ReturnValues: "ALL_OLD",
1177 }),
1178 );
1179 yield* deleteOverflow(str(previous.Attributes, "ref"));
1180 return request.value;
1181 }
1182 yield* register(request.stack, request.stage, snapshot, lease);
1183 yield* putPayload(
1184 {
1185 PK: stagePk(request.stack, request.stage),
1186 SK: OUTPUT,
1187 stack: request.stack,
1188 stage: request.stage,
1189 },
1190 JSON.stringify(encodeState(request.value)),
1191 {},
1192 );
1193 return request.value;
1194 }),
1195 ),
1196 };
1197
1198 /**
1199 * Run `body` holding the stage's lease, released as soon as it finishes
1200 * rather than when the store closes.
1201 */
1202 const exclusive = <A, R>(
1203 stack: string,
1204 stage: string,
1205 body: (lease: HeldLease) => Effect.Effect<A, StateStoreError, R>,
1206 ) =>
1207 Effect.scoped(
1208 Effect.gen(function* () {
1209 const key = `${stack}/${stage}`;
1210 const scope = yield* Effect.scope;
1211 const lease = yield* acquireLease(stack, stage, scope);
1212 yield* Scope.addFinalizer(
1213 scope,
1214 Effect.sync(() => {
1215 snapshots.entries.delete(key);
1216 registered.entries.delete(key);
1217 }),
1218 );
1219 return yield* body(lease);
1220 }),
1221 );
1222
1223 /**
1224 * Make the stage hold exactly `contents`, check it does, and enter it in
1225 * the catalog: the commit that makes DynamoDB the stage's store.
1226 */
1227 const commitStage = Effect.fn(function* (
1228 stack: string,
1229 stage: string,
1230 lease: HeldLease,
1231 contents: StageContents,
1232 migratedFrom: string,
1233 beforeCommit: Effect.Effect<void, StateStoreError> = Effect.void,
1234 ) {
1235 yield* lease.checkLive;
1236 const pk = stagePk(stack, stage);
1237 const wanted = new Set(yield* Effect.forEach(Array.from(contents.records.keys()), recordSk));
1238 if (contents.output !== undefined) wanted.add(OUTPUT);
1239 // Items an earlier, interrupted copy left that the source no longer has.
1240 const stale = (yield* queryPartition(pk, undefined, "PK, SK, #ref")).filter((item) => {
1241 const sk = str(item, "SK");
1242 return sk !== LEASE && sk !== undefined && !wanted.has(sk);
1243 });
1244 yield* deleteItems(stale.map((item) => ({ PK: str(item, "PK")!, SK: str(item, "SK")! })));
1245 yield* Effect.forEach(stale, (item) => deleteOverflow(str(item, "ref")), { discard: true });
1246 yield* Effect.forEach(
1247 Array.from(contents.records),
1248 ([fqn, text]) =>
1249 Effect.flatMap(recordSk(fqn), (sk) =>
1250 putPayload({ PK: pk, SK: sk, stack, stage }, text, { fqn: S(fqn) }),
1251 ),
1252 { concurrency: 16, discard: true },
1253 );
1254 if (contents.output !== undefined) {
1255 yield* putPayload({ PK: pk, SK: OUTPUT, stack, stage }, contents.output, {});
1256 }
1257 // Read back what landed before it becomes the stage's state.
1258 const copied = yield* readStage(stack, stage);
1259 const mismatched = [
1260 ...Array.from(contents.records).filter(([fqn, text]) => copied.records.get(fqn) !== text),
1261 ...Array.from(copied.records.keys()).filter((fqn) => !contents.records.has(fqn)),
1262 ];
1263 if (mismatched.length > 0 || copied.output !== contents.output) {
1264 return yield* new StateStoreError({
1265 message: `DynamoDB copy of '${stack}/${stage}' does not match its source (${mismatched.length} records differ${copied.output !== contents.output ? ", output differs" : ""})`,
1266 });
1267 }
1268 yield* beforeCommit;
1269 const migratedAt = new Date(yield* Clock.currentTimeMillis).toISOString();
1270 yield* writeUnderLease(stack, stage, lease, (table) => ({
1271 Put: {
1272 TableName: table,
1273 Item: {
1274 ...catalogKey(stack, stage),
1275 stack: S(stack),
1276 stage: S(stage),
1277 migratedFrom: S(migratedFrom),
1278 migratedAt: S(migratedAt),
1279 },
1280 },
1281 }));
1282 });
1283
1284 /** Mark the stage as moved back: the S3 store is its store again. */
1285 const markRolledBack = (stack: string, stage: string, lease: HeldLease) =>
1286 writeUnderLease(stack, stage, lease, (table) => ({
1287 Update: {
1288 TableName: table,
1289 Key: catalogKey(stack, stage),
1290 UpdateExpression: "SET #status = :status",
1291 ExpressionAttributeNames: { "#status": "status" },
1292 ExpressionAttributeValues: { ":status": S("rolled-back") },
1293 },
1294 }));
1295
1296 const service: DynamoDBStateService = {
1297 ...state,
1298 migrate: (request) =>
1299 (request.rollback
1300 ? rollbackToS3(service, request)
1301 : migrateToDynamoDB(service, request)
1302 ).pipe(Effect.provideContext(context)),
1303 migration: {
1304 names,
1305 provision: names.pipe(
1306 Effect.tap((resolved) => run(provision(resolved.table, resolved.bucket, resolved.region))),
1307 ),
1308 catalog: () => catalog(),
1309 status: catalogStatus,
1310 exclusive,
1311 readStage,
1312 commitStage,
1313 markRolledBack,
1314 deleteStageItems: (stack, stage, lease) =>
1315 lease.checkLive.pipe(Effect.andThen(deleteStageItems(stack, stage))),
1316 },
1317 };
1318 return service;
1319 });
1320
1321 /** Why each item of a cancelled transaction failed, in request order. */
1322 const cancellationCodes = (error: dynamodb.TransactionCanceledException) =>
1323 // The generated schema leaves the reason list untyped.
1324 ((error.CancellationReasons ?? []) as ReadonlyArray<dynamodb.CancellationReason>).map(
1325 (reason) => reason.Code,
1326 );
1327
1328 /** A stage's snapshot: its records and its catalog status when loaded. */
1329 interface Snapshot {
1330 readonly records: Map<string, StoredRecord>;
1331 status: CatalogStatus | undefined;
1332 }
1333
1334 const statusOf = (item: Item | undefined): CatalogStatus | undefined =>
1335 item === undefined ? undefined : ((str(item, "status") as CatalogStatus | undefined) ?? "active");
1336
1337 /**
1338 * The DynamoDB store, with the operations `alchemy state migrate` moves
1339 * stages with.
1340 */
1341 export interface DynamoDBStateService extends StateService {
1342 readonly migration: {
1343 /** The table, overflow bucket and S3 store this store addresses. */
1344 readonly names: Effect.Effect<
1345 { table: string; bucket: string; region: string; legacy: LegacyLocation | undefined },
1346 StateStoreError
1347 >;
1348 /** {@link names}, after creating the table and overflow bucket if missing. */
1349 readonly provision: Effect.Effect<
1350 { table: string; bucket: string; region: string; legacy: LegacyLocation | undefined },
1351 StateStoreError
1352 >;
1353 readonly catalog: () => Effect.Effect<
1354 ReadonlyArray<{ stack: string; stage: string; status: CatalogStatus }>,
1355 StateStoreError
1356 >;
1357 readonly status: (
1358 stack: string,
1359 stage: string,
1360 ) => Effect.Effect<CatalogStatus | undefined, StateStoreError>;
1361 readonly exclusive: <A, R>(
1362 stack: string,
1363 stage: string,
1364 body: (lease: HeldLease) => Effect.Effect<A, StateStoreError, R>,
1365 ) => Effect.Effect<A, StateStoreError, Exclude<R, Scope.Scope>>;
1366 readonly readStage: (
1367 stack: string,
1368 stage: string,
1369 ) => Effect.Effect<StageContents, StateStoreError>;
1370 readonly commitStage: (
1371 stack: string,
1372 stage: string,
1373 lease: HeldLease,
1374 contents: StageContents,
1375 migratedFrom: string,
1376 beforeCommit?: Effect.Effect<void, StateStoreError>,
1377 ) => Effect.Effect<void, StateStoreError>;
1378 readonly markRolledBack: (
1379 stack: string,
1380 stage: string,
1381 lease: HeldLease,
1382 ) => Effect.Effect<void, StateStoreError>;
1383 readonly deleteStageItems: (
1384 stack: string,
1385 stage: string,
1386 lease: HeldLease,
1387 ) => Effect.Effect<void, StateStoreError>;
1388 };
1389 }
1390
1391 /** The table's description, or `undefined` when it doesn't exist. */
1392 const describeStateTable = (table: string) =>
1393 dynamodb.describeTable({ TableName: table }).pipe(
1394 Effect.map((result) => result.Table),
1395 Effect.catchTag("ResourceNotFoundException", () => Effect.void),
1396 );
1397
1398 const validateKeySchema = (table: dynamodb.TableDescription) => {
1399 const keys = table.KeySchema ?? [];
1400 const hash = keys.find((key) => key.KeyType === "HASH")?.AttributeName;
1401 const range = keys.find((key) => key.KeyType === "RANGE")?.AttributeName;
1402 return hash === "PK" && range === "SK"
1403 ? Effect.void
1404 : Effect.fail(
1405 new StateStoreError({
1406 message: `DynamoDB state table '${table.TableName}' must have partition key PK and sort key SK (found ${hash ?? "none"} and ${range ?? "none"})`,
1407 }),
1408 );
1409 };
1410
1411 class TableNotActive extends Data.TaggedError("TableNotActive") {}
1412
1413 const waitForTableActive = (table: string) =>
1414 dynamodb.describeTable({ TableName: table }).pipe(
1415 Effect.flatMap((result) =>
1416 result.Table?.TableStatus === "ACTIVE"
1417 ? Effect.succeed(result.Table)
1418 : Effect.fail(new TableNotActive()),
1419 ),
1420 Effect.retry({
1421 // A just-created table can briefly describe as missing.
1422 while: (error) =>
1423 error._tag === "TableNotActive" || error._tag === "ResourceNotFoundException",
1424 schedule: Schedule.spaced("1 second"),
1425 times: 60,
1426 }),
1427 Effect.catchTag("TableNotActive", () =>
1428 Effect.fail(
1429 new StateStoreError({ message: `DynamoDB state table '${table}' never became ACTIVE` }),
1430 ),
1431 ),
1432 );
1433
1434 /**
1435 * Observe-then-ensure the state table: create it if missing (tolerating
1436 * create races), with deletion protection and, given `kmsKeyId`, that KMS
1437 * key; wait for it to become ACTIVE, and enable point-in-time recovery.
1438 */
1439 const ensureStateTable = Effect.fn(function* (table: string, kmsKeyId?: string) {
1440 const observed = yield* describeStateTable(table);
1441 if (observed === undefined) {
1442 yield* Effect.logInfo(`DynamoDB state store: creating table ${table}`);
1443 yield* dynamodb
1444 .createTable({
1445 TableName: table,
1446 AttributeDefinitions: [
1447 { AttributeName: "PK", AttributeType: "S" },
1448 { AttributeName: "SK", AttributeType: "S" },
1449 ],
1450 KeySchema: [
1451 { AttributeName: "PK", KeyType: "HASH" },
1452 { AttributeName: "SK", KeyType: "RANGE" },
1453 ],
1454 BillingMode: "PAY_PER_REQUEST",
1455 // The table holds the state of every stack on the account.
1456 DeletionProtectionEnabled: true,
1457 ...(kmsKeyId === undefined
1458 ? {}
1459 : { SSESpecification: { Enabled: true, SSEType: "KMS", KMSMasterKeyId: kmsKeyId } }),
1460 })
1461 // Another first caller is creating it.
1462 .pipe(Effect.catchTag("ResourceInUseException", () => Effect.void));
1463 } else {
1464 yield* validateKeySchema(observed);
1465 }
1466 yield* waitForTableActive(table);
1467
1468 const backups = yield* dynamodb.describeContinuousBackups({ TableName: table });
1469 if (
1470 backups.ContinuousBackupsDescription?.PointInTimeRecoveryDescription
1471 ?.PointInTimeRecoveryStatus !== "ENABLED"
1472 ) {
1473 yield* dynamodb
1474 .updateContinuousBackups({
1475 TableName: table,
1476 PointInTimeRecoverySpecification: { PointInTimeRecoveryEnabled: true },
1477 })
1478 .pipe(
1479 // Backups become available shortly after the table is ACTIVE.
1480 Effect.retry({
1481 while: (error) => error._tag === "ContinuousBackupsUnavailableException",
1482 schedule: Schedule.spaced("2 seconds"),
1483 times: 15,
1484 }),
1485 );
1486 }
1487 });
1488
1489 /**
1490 * Create the state table, or converge an existing one: what first use of
1491 * {@link dynamoDBState} does.
1492 */
1493 export const provisionStateTable = (
1494 table: string,
1495 kmsKeyId?: string,
1496 ): Effect.Effect<void, StateStoreError, AwsDeps> =>
1497 ensureStateTable(table, kmsKeyId).pipe(Effect.mapError(toError));
1498
1499 /** Whether a state table exists. */
1500 export const stateTableExists = (table: string) =>
1501 describeStateTable(table).pipe(Effect.map((observed) => observed !== undefined));