The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import { createHash } from "node:crypto"; |
| 2 | import type { Credentials } from "@distilled.cloud/aws/Credentials"; |
| 3 | import * as dynamodb from "@distilled.cloud/aws/dynamodb"; |
| 4 | import type { Region } from "@distilled.cloud/aws/Region"; |
| 5 | import * as s3 from "@distilled.cloud/aws/s3"; |
| 6 | import { STATE_STORE_VERSION } from "alchemy/State/HttpStateApi"; |
| 7 | import * as Provisioning from "alchemy/State/Provisioning"; |
| 8 | import { |
| 9 | type PersistedState, |
| 10 | State, |
| 11 | type StateService, |
| 12 | StateStoreError, |
| 13 | } from "alchemy/State/State"; |
| 14 | import { encodeState, reviveState } from "alchemy/State/StateEncoding"; |
| 15 | import { recordStateStoreInit } from "alchemy/Telemetry/Metrics"; |
| 16 | import * as Arr from "effect/Array"; |
| 17 | import * as Clock from "effect/Clock"; |
| 18 | import * as Data from "effect/Data"; |
| 19 | import * as Duration from "effect/Duration"; |
| 20 | import * as Effect from "effect/Effect"; |
| 21 | import type { HttpClient } from "effect/http/HttpClient"; |
| 22 | import * as Layer from "effect/Layer"; |
| 23 | import * as Schedule from "effect/Schedule"; |
| 24 | import * as Scope from "effect/Scope"; |
| 25 | import * as Semaphore from "effect/Semaphore"; |
| 26 | import * as Stream from "effect/Stream"; |
| 27 | import { AWSEnvironment } from "../Environment.ts"; |
| 28 | import { |
| 29 | fenceLegacyStage, |
| 30 | HELD_PREFIX, |
| 31 | hasLegacyStage, |
| 32 | type LegacyLocation, |
| 33 | listLegacyStacks, |
| 34 | listLegacyStages, |
| 35 | normalizePrefix, |
| 36 | readLegacyOutput, |
| 37 | type StageContents, |
| 38 | } from "./LegacyLayout.ts"; |
| 39 | import { type CatalogStatus, migrateToDynamoDB, ownsStage, rollbackToS3 } from "./Migration.ts"; |
| 40 | import { |
| 41 | createStateBucketName, |
| 42 | provisionStateBucket, |
| 43 | stateBucketExists, |
| 44 | stateEnvironment, |
| 45 | withoutSdkDebugLogs, |
| 46 | } from "./State.ts"; |
| 47 | |
| 48 | export interface DynamoDBStateOptions { |
| 49 | /** |
| 50 | * A store other than the account's default, by logical name: its table is |
| 51 | * `alchemy-state-<name>` and its overflow bucket |
| 52 | * `alchemy-state-overflow-<name>-<accountId>-<region>-an`. The table is |
| 53 | * created on first use, except in CI, where `alchemy state bootstrap |
| 54 | * --backend aws --name <name>` provisions it ahead. |
| 55 | * |
| 56 | * @default the account's default store, table `alchemy-state` |
| 57 | */ |
| 58 | name?: string; |
| 59 | /** |
| 60 | * A full table name instead of one derived from |
| 61 | * {@link DynamoDBStateOptions.name}, e.g. an existing table. It must have |
| 62 | * a string partition key `PK` and a string sort key `SK`. |
| 63 | */ |
| 64 | tableName?: string; |
| 65 | /** |
| 66 | * A full name for the S3 bucket that holds records too large for a |
| 67 | * DynamoDB item, instead of one derived from |
| 68 | * {@link DynamoDBStateOptions.name}. Created, with the S3 state store's |
| 69 | * secure defaults, with the table. |
| 70 | * |
| 71 | * @default `alchemy-state-overflow-<accountId>-<region>-an` |
| 72 | */ |
| 73 | overflowBucketName?: string; |
| 74 | /** |
| 75 | * The S3 state store whose stages this store takes over. A stage that is |
| 76 | * still there and not yet in DynamoDB is refused until `alchemy state |
| 77 | * migrate` moves it; `false` skips the check, and the S3 access it needs. |
| 78 | * |
| 79 | * @default the S3 store of the same name, `alchemy-state-<name>-<accountId>-<region>-an` |
| 80 | */ |
| 81 | migrateFrom?: { readonly bucketName?: string; readonly prefix?: string } | false; |
| 82 | /** |
| 83 | * A customer-managed KMS key (id, ARN or alias) that encrypts the table |
| 84 | * and the overflow bucket. Applied when the store creates them. |
| 85 | * |
| 86 | * @default AWS-owned keys |
| 87 | */ |
| 88 | kmsKeyId?: string; |
| 89 | /** |
| 90 | * Size in bytes of a record's encoded state above which it is stored in |
| 91 | * the overflow bucket, with the DynamoDB item holding a reference to it. |
| 92 | * A DynamoDB item holds at most 400 KB. |
| 93 | * |
| 94 | * @default 307200 (300 KiB) |
| 95 | */ |
| 96 | overflowThreshold?: number; |
| 97 | } |
| 98 | |
| 99 | /** The backend name `alchemy state ... --backend` addresses this store by. */ |
| 100 | export const DYNAMODB_STATE_BACKEND = "aws"; |
| 101 | |
| 102 | /** Context required by the distilled DynamoDB and S3 operations. */ |
| 103 | type AwsDeps = Credentials | HttpClient | Region; |
| 104 | |
| 105 | type Item = { [key: string]: dynamodb.AttributeValue | undefined }; |
| 106 | |
| 107 | const DEFAULT_OVERFLOW_THRESHOLD = 300 * 1024; |
| 108 | |
| 109 | /** Maximum number of requests DynamoDB accepts in one BatchWriteItem call. */ |
| 110 | const BATCH_WRITE_SIZE = 25; |
| 111 | |
| 112 | /** Maximum number of keys S3 accepts in one DeleteObjects call. */ |
| 113 | const S3_DELETE_BATCH_SIZE = 1000; |
| 114 | |
| 115 | /** Backoff for the writes a BatchWriteItem call leaves unprocessed. */ |
| 116 | const unprocessedBackoff = Schedule.exponential("200 millis").pipe( |
| 117 | Schedule.modifyDelay(({ duration }) => |
| 118 | Effect.succeed(Duration.min(duration, Duration.seconds(5))), |
| 119 | ), |
| 120 | Schedule.upTo({ times: 8 }), |
| 121 | ); |
| 122 | |
| 123 | /** How long a lease stays valid without a refresh. */ |
| 124 | const LEASE_TTL_MS = 60_000; |
| 125 | |
| 126 | /** How long a passing lease check is trusted. A failure is never cached. */ |
| 127 | const LEASE_CHECK_TTL_MS = 5_000; |
| 128 | |
| 129 | /** Sort key prefix of resource and action records. */ |
| 130 | const RECORD = "RECORD#"; |
| 131 | /** Sort key of a stage's stack output. */ |
| 132 | const OUTPUT = "OUTPUT"; |
| 133 | /** Sort key of a stage's deploy lease. */ |
| 134 | const LEASE = "LEASE"; |
| 135 | /** Partition key of the stage catalog. */ |
| 136 | const CATALOG = "CATALOG"; |
| 137 | |
| 138 | /** Escape the key separator out of a stack or stage name. */ |
| 139 | const part = (name: string) => name.replaceAll("%", "%25").replaceAll("#", "%23"); |
| 140 | |
| 141 | const stagePk = (stack: string, stage: string) => `STAGE#${part(stack)}#${part(stage)}`; |
| 142 | /** The catalog sort key prefix of a stack's stages. */ |
| 143 | const catalogPrefix = (stack: string) => `STAGE#${part(stack)}#`; |
| 144 | const catalogSk = (stack: string, stage: string) => `${catalogPrefix(stack)}${part(stage)}`; |
| 145 | |
| 146 | const S = (value: string): dynamodb.AttributeValue => ({ S: value }); |
| 147 | const N = (value: number): dynamodb.AttributeValue => ({ N: String(value) }); |
| 148 | const str = (item: Item | undefined, name: string) => item?.[name]?.S; |
| 149 | |
| 150 | /** |
| 151 | * Build the default table name. Table names are unique per account and |
| 152 | * region, so they need no account or region suffix. |
| 153 | */ |
| 154 | export const createStateTableName = (name?: string) => |
| 155 | `alchemy-state${name === undefined ? "" : `-${name}`}`; |
| 156 | |
| 157 | /** S3's limit on the length of a bucket name. */ |
| 158 | const MAX_BUCKET_NAME = 63; |
| 159 | |
| 160 | /** |
| 161 | * Build the default overflow bucket name, in the account-regional namespace. |
| 162 | * A store name too long to fit is shortened, with a hash of the full name |
| 163 | * keeping it unique. |
| 164 | */ |
| 165 | export const createOverflowBucketName = (accountId: string, region: string, name?: string) => { |
| 166 | const suffix = `${accountId}-${region}-an`; |
| 167 | const full = `alchemy-state-overflow-${name === undefined ? "" : `${name}-`}${suffix}`; |
| 168 | if (name === undefined || full.length <= MAX_BUCKET_NAME) return full.toLowerCase(); |
| 169 | const hash = createHash("sha256").update(name).digest("hex").slice(0, 8); |
| 170 | const room = MAX_BUCKET_NAME - `alchemy-state-overflow--${hash}-${suffix}`.length; |
| 171 | return `alchemy-state-overflow-${name.slice(0, Math.max(0, room)).replace(/-+$/, "")}-${hash}-${suffix}` |
| 172 | .replace("--", "-") |
| 173 | .toLowerCase(); |
| 174 | }; |
| 175 | |
| 176 | const toError = (cause: unknown) => |
| 177 | new StateStoreError({ |
| 178 | message: |
| 179 | cause instanceof Error ? cause.message : `DynamoDB state store error: ${String(cause)}`, |
| 180 | cause: cause instanceof Error ? cause : undefined, |
| 181 | }); |
| 182 | |
| 183 | /** |
| 184 | * State store backed by an AWS DynamoDB table. |
| 185 | * |
| 186 | * Every stack and stage on the account shares one on-demand table with a |
| 187 | * string partition key `PK` and sort key `SK`: |
| 188 | * |
| 189 | * ``` |
| 190 | * STAGE#{stack}#{stage} RECORD#{sha256(fqn)} one resource's state |
| 191 | * STAGE#{stack}#{stage} OUTPUT the stack's output |
| 192 | * STAGE#{stack}#{stage} LEASE the deploy lease |
| 193 | * CATALOG STAGE#{stack}#{stage} the stage catalog |
| 194 | * ``` |
| 195 | * |
| 196 | * A deploy takes the stage's lease, then loads every record of the stage |
| 197 | * with one paginated Query and serves reads from that snapshot. Writes go |
| 198 | * to DynamoDB immediately, one item each. A record larger than |
| 199 | * `overflowThreshold` is stored in an S3 overflow bucket, and its item |
| 200 | * holds the object's key. |
| 201 | * |
| 202 | * Concurrent deploys of one stack and stage contend for the lease, a |
| 203 | * conditional put. The second deploy fails immediately instead of mixing |
| 204 | * rows. The holder refreshes the lease until the store closes. A killed |
| 205 | * process stops refreshing, the lease expires, and the next deploy can |
| 206 | * take it. |
| 207 | * |
| 208 | * The table and overflow bucket are created on the first state operation |
| 209 | * if they do not already exist, the table with deletion protection and |
| 210 | * point-in-time recovery enabled. |
| 211 | * |
| 212 | * ### Using the DynamoDB State Store |
| 213 | * Pass `AWS.state()` as the `state` option of a Stack. |
| 214 | * |
| 215 | * **Example:** Default table |
| 216 | * ```typescript |
| 217 | * import * as Alchemy from "alchemy"; |
| 218 | * import * as AWS from "@alchemy.run/aws"; |
| 219 | * |
| 220 | * const Stack = Alchemy.Stack( |
| 221 | * "my-stack", |
| 222 | * { providers: AWS.providers(), state: AWS.state() }, |
| 223 | * Effect.gen(function* () { |
| 224 | * // ... |
| 225 | * }), |
| 226 | * ); |
| 227 | * ``` |
| 228 | * |
| 229 | * **Example:** A named store |
| 230 | * ```typescript |
| 231 | * const Stack = Alchemy.Stack( |
| 232 | * "my-stack", |
| 233 | * { |
| 234 | * providers: AWS.providers(), |
| 235 | * state: AWS.state({ name: "team-a" }), // table alchemy-state-team-a |
| 236 | * }, |
| 237 | * Effect.gen(function* () { |
| 238 | * // ... |
| 239 | * }), |
| 240 | * ); |
| 241 | * ``` |
| 242 | * |
| 243 | * ### Provisioning the Store in CI |
| 244 | * In CI a missing table fails the run. Create it, and its overflow bucket, |
| 245 | * once ahead of time: |
| 246 | * |
| 247 | * ```sh |
| 248 | * alchemy state bootstrap --backend aws --profile <your-ci-profile> |
| 249 | * ``` |
| 250 | * |
| 251 | */ |
| 252 | export const dynamoDBState = (options: DynamoDBStateOptions = {}) => |
| 253 | Layer.effect( |
| 254 | State, |
| 255 | Effect.gen(function* () { |
| 256 | const scope = yield* Effect.scope; |
| 257 | const context = yield* Effect.context<AwsDeps | AWSEnvironment>(); |
| 258 | |
| 259 | // Bind the leases to this layer's scope. The cached effect runs on |
| 260 | // the first state operation, which may sit in a shorter scope. |
| 261 | const make = Scope.provide(makeDynamoDBState(options), scope).pipe( |
| 262 | recordStateStoreInit, |
| 263 | Effect.orDie, |
| 264 | Effect.provideContext(context), |
| 265 | ); |
| 266 | |
| 267 | return yield* Effect.cached(make); |
| 268 | }), |
| 269 | ).pipe(Layer.provideMerge(stateEnvironment()), Layer.orDie); |
| 270 | |
| 271 | interface StoredRecord { |
| 272 | /** The encoded state, as JSON. */ |
| 273 | readonly text: string; |
| 274 | } |
| 275 | |
| 276 | /** A stage lease this store holds. */ |
| 277 | export interface HeldLease { |
| 278 | readonly token: string; |
| 279 | readonly checkLive: Effect.Effect<void, StateStoreError>; |
| 280 | } |
| 281 | |
| 282 | /** |
| 283 | * A passing check is trusted for `ttlMs`, so a burst of operations does |
| 284 | * one round-trip. A failing check is never cached. |
| 285 | */ |
| 286 | const amortizeCheck = ( |
| 287 | checkLive: Effect.Effect<void, StateStoreError>, |
| 288 | ttlMs: number, |
| 289 | lastOkAt: number | undefined, |
| 290 | ): Effect.Effect<void, StateStoreError> => |
| 291 | Effect.suspend(() => { |
| 292 | if (lastOkAt !== undefined && Date.now() - lastOkAt < ttlMs) { |
| 293 | return Effect.void; |
| 294 | } |
| 295 | return checkLive.pipe( |
| 296 | Effect.tap(() => |
| 297 | Effect.sync(() => { |
| 298 | lastOkAt = Date.now(); |
| 299 | }), |
| 300 | ), |
| 301 | ); |
| 302 | }); |
| 303 | |
| 304 | /** |
| 305 | * One cached effect per key. A failed effect is dropped rather than cached, |
| 306 | * so the next call retries it. |
| 307 | */ |
| 308 | const keyedCache = <A>() => { |
| 309 | const mutex = Semaphore.makeUnsafe(1); |
| 310 | const entries = new Map<string, Effect.Effect<A, StateStoreError>>(); |
| 311 | return { |
| 312 | entries, |
| 313 | get: (key: string, make: Effect.Effect<A, StateStoreError>) => |
| 314 | Semaphore.withPermits( |
| 315 | mutex, |
| 316 | 1, |
| 317 | )( |
| 318 | Effect.gen(function* () { |
| 319 | const existing = entries.get(key); |
| 320 | if (existing !== undefined) return existing; |
| 321 | const cached = yield* Effect.cached( |
| 322 | make.pipe( |
| 323 | Effect.tapError(() => |
| 324 | Effect.sync(() => { |
| 325 | if (entries.get(key) === cached) entries.delete(key); |
| 326 | }), |
| 327 | ), |
| 328 | ), |
| 329 | ); |
| 330 | entries.set(key, cached); |
| 331 | return cached; |
| 332 | }), |
| 333 | ).pipe(Effect.flatMap((effect) => effect)), |
| 334 | }; |
| 335 | }; |
| 336 | |
| 337 | /** |
| 338 | * Construct a DynamoDB-backed {@link StateService}. Leases are bound to the |
| 339 | * ambient scope; callers that need them to outlive the first operation |
| 340 | * bind this effect to a longer scope with `Scope.provide`. |
| 341 | * |
| 342 | * Construction itself never touches AWS: resolving the environment and |
| 343 | * ensuring the table exists run once, on the first state operation. |
| 344 | */ |
| 345 | export const makeDynamoDBState = (options: DynamoDBStateOptions = {}) => |
| 346 | makeStore(options).pipe(withoutSdkDebugLogs); |
| 347 | |
| 348 | const makeStore = Effect.fn(function* (options: DynamoDBStateOptions) { |
| 349 | const leaseScope = yield* Effect.scope; |
| 350 | // Captured under `withoutSdkDebugLogs` (below), so every store call runs |
| 351 | // with the raised floor: request payloads are state, and state holds |
| 352 | // secrets. |
| 353 | const context = yield* Effect.context<AwsDeps | AWSEnvironment>(); |
| 354 | const run = <A, E>(effect: Effect.Effect<A, E, AwsDeps>): Effect.Effect<A, StateStoreError> => |
| 355 | effect.pipe(Effect.provideContext(context), Effect.mapError(toError)); |
| 356 | |
| 357 | const threshold = options.overflowThreshold ?? DEFAULT_OVERFLOW_THRESHOLD; |
| 358 | // Only the stack's config describes a store with its own names or key, so |
| 359 | // `--backend aws` can't bootstrap it. |
| 360 | const configured = |
| 361 | options.tableName !== undefined || |
| 362 | options.overflowBucketName !== undefined || |
| 363 | options.kmsKeyId !== undefined; |
| 364 | const label = "DynamoDB"; |
| 365 | |
| 366 | // The table, overflow bucket and S3 store this store addresses. |
| 367 | const names = yield* Effect.cached( |
| 368 | Effect.gen(function* () { |
| 369 | const { accountId, region } = yield* AWSEnvironment.current; |
| 370 | if (options.name !== undefined) yield* Provisioning.storeName(options.name); |
| 371 | const table = options.tableName ?? createStateTableName(options.name); |
| 372 | const bucket = |
| 373 | options.overflowBucketName ?? createOverflowBucketName(accountId, region, options.name); |
| 374 | const legacy: LegacyLocation | undefined = |
| 375 | options.migrateFrom === false |
| 376 | ? undefined |
| 377 | : { |
| 378 | bucket: |
| 379 | options.migrateFrom?.bucketName ?? |
| 380 | createStateBucketName(accountId, region, options.name), |
| 381 | prefix: normalizePrefix(options.migrateFrom?.prefix), |
| 382 | }; |
| 383 | return { table, bucket, region, legacy }; |
| 384 | }).pipe(Effect.provideContext(context), Effect.mapError(toError)), |
| 385 | ); |
| 386 | |
| 387 | const provision = (table: string, bucket: string, region: string) => |
| 388 | Effect.all( |
| 389 | [ |
| 390 | ensureStateTable(table, options.kmsKeyId), |
| 391 | provisionStateBucket(bucket, region, options.kmsKeyId), |
| 392 | ], |
| 393 | { concurrency: 2, discard: true }, |
| 394 | ); |
| 395 | |
| 396 | // Ensure the table and overflow bucket exist on the first operation, so a |
| 397 | // missing one fails the run before it changes anything. |
| 398 | const target = yield* Effect.cached( |
| 399 | Effect.gen(function* () { |
| 400 | const { table, bucket, region, legacy } = yield* names; |
| 401 | const [observed, bucketExists] = yield* Effect.all( |
| 402 | [describeStateTable(table), stateBucketExists(bucket)], |
| 403 | { concurrency: 2 }, |
| 404 | ); |
| 405 | if (observed === undefined) { |
| 406 | yield* Provisioning.deployMissing( |
| 407 | { |
| 408 | backend: DYNAMODB_STATE_BACKEND, |
| 409 | name: options.name, |
| 410 | configured, |
| 411 | label: `DynamoDB state table '${table}'`, |
| 412 | }, |
| 413 | provision(table, bucket, region), |
| 414 | ); |
| 415 | } else { |
| 416 | yield* validateKeySchema(observed); |
| 417 | if (observed.TableStatus !== "ACTIVE") yield* waitForTableActive(table); |
| 418 | if (!bucketExists) { |
| 419 | yield* Provisioning.deployMissing( |
| 420 | { |
| 421 | backend: DYNAMODB_STATE_BACKEND, |
| 422 | name: options.name, |
| 423 | configured, |
| 424 | label: `DynamoDB state overflow bucket '${bucket}'`, |
| 425 | }, |
| 426 | provisionStateBucket(bucket, region, options.kmsKeyId), |
| 427 | ); |
| 428 | } |
| 429 | } |
| 430 | return { table, bucket, region, legacy }; |
| 431 | }).pipe(Effect.provideContext(context), Effect.mapError(toError)), |
| 432 | ); |
| 433 | |
| 434 | const use = <A, E>( |
| 435 | f: (target: { |
| 436 | table: string; |
| 437 | bucket: string; |
| 438 | region: string; |
| 439 | legacy: LegacyLocation | undefined; |
| 440 | }) => Effect.Effect<A, E, AwsDeps>, |
| 441 | ): Effect.Effect<A, StateStoreError> => target.pipe(Effect.flatMap((t) => run(f(t)))); |
| 442 | |
| 443 | const hashFqn = (fqn: string) => |
| 444 | Effect.sync(() => createHash("sha256").update(fqn).digest("hex")); |
| 445 | |
| 446 | const recordSk = (fqn: string) => hashFqn(fqn).pipe(Effect.map((hash) => `${RECORD}${hash}`)); |
| 447 | |
| 448 | const blobPrefix = (stack: string, stage: string) => |
| 449 | `${encodeURIComponent(stack)}/${encodeURIComponent(stage)}/`; |
| 450 | |
| 451 | /** The encoded text of an item: inline, or read from the overflow bucket. */ |
| 452 | const itemText = Effect.fn(function* (item: Item) { |
| 453 | const inline = str(item, "data"); |
| 454 | if (inline !== undefined) return inline; |
| 455 | const ref = str(item, "ref"); |
| 456 | if (ref === undefined) { |
| 457 | return yield* new StateStoreError({ |
| 458 | message: `DynamoDB state item ${str(item, "PK")}/${str(item, "SK")} has neither data nor ref`, |
| 459 | }); |
| 460 | } |
| 461 | const text = yield* use(({ bucket }) => |
| 462 | s3.getObject({ Bucket: bucket, Key: ref }).pipe( |
| 463 | Effect.flatMap((result) => |
| 464 | result.Body === undefined |
| 465 | ? Effect.void |
| 466 | : result.Body.pipe(Stream.decodeText, Stream.mkString), |
| 467 | ), |
| 468 | Effect.catchTag("NoSuchKey", () => Effect.void), |
| 469 | ), |
| 470 | ); |
| 471 | // A missing body is corruption, never a missing resource. |
| 472 | if (text === undefined) { |
| 473 | return yield* new StateStoreError({ |
| 474 | message: `DynamoDB state item ${str(item, "PK")}/${str(item, "SK")} references missing overflow object '${ref}'`, |
| 475 | }); |
| 476 | } |
| 477 | return text; |
| 478 | }); |
| 479 | |
| 480 | const parse = <T>(text: string, what: string) => |
| 481 | Effect.try({ |
| 482 | try: () => JSON.parse(text, reviveState) as T, |
| 483 | catch: (cause) => |
| 484 | new StateStoreError({ |
| 485 | message: `Failed to parse DynamoDB state ${what}`, |
| 486 | cause: cause instanceof Error ? cause : undefined, |
| 487 | }), |
| 488 | }); |
| 489 | |
| 490 | /** |
| 491 | * Write an item holding `value`, inline or as an overflow reference. |
| 492 | * The overflow object is uploaded before the item that references it is |
| 493 | * committed, and the object the item referenced before is deleted after. |
| 494 | */ |
| 495 | const putPayload = Effect.fn(function* ( |
| 496 | key: { PK: string; SK: string; stack: string; stage: string }, |
| 497 | text: string, |
| 498 | attributes: Item, |
| 499 | ) { |
| 500 | const bytes = yield* Effect.sync(() => new TextEncoder().encode(text).length); |
| 501 | let payload: Item; |
| 502 | if (bytes > threshold) { |
| 503 | // A fresh key per write, so the committed item's object is never |
| 504 | // overwritten in place. |
| 505 | const ref = `${blobPrefix(key.stack, key.stage)}${encodeURIComponent(key.SK)}-${yield* Effect.sync(() => crypto.randomUUID())}.json`; |
| 506 | yield* use(({ bucket }) => |
| 507 | s3.putObject({ |
| 508 | Bucket: bucket, |
| 509 | Key: ref, |
| 510 | Body: text, |
| 511 | ContentType: "application/json", |
| 512 | }), |
| 513 | ); |
| 514 | payload = { ref: S(ref) }; |
| 515 | } else { |
| 516 | payload = { data: S(text) }; |
| 517 | } |
| 518 | const previous = yield* use(({ table }) => |
| 519 | dynamodb.putItem({ |
| 520 | TableName: table, |
| 521 | Item: { PK: S(key.PK), SK: S(key.SK), ...attributes, ...payload }, |
| 522 | ReturnValues: "ALL_OLD", |
| 523 | }), |
| 524 | ); |
| 525 | yield* deleteOverflow(str(previous.Attributes, "ref"), str(payload, "ref")); |
| 526 | }); |
| 527 | |
| 528 | /** Delete an overflow object no item references any more. Best effort. */ |
| 529 | const deleteOverflow = (ref: string | undefined, keep?: string) => |
| 530 | ref === undefined || ref === keep |
| 531 | ? Effect.void |
| 532 | : use(({ bucket }) => s3.deleteObject({ Bucket: bucket, Key: ref })).pipe(Effect.ignore); |
| 533 | |
| 534 | /** Delete items in batches, retrying the ones DynamoDB leaves unprocessed. */ |
| 535 | const deleteItems = (keys: ReadonlyArray<{ PK: string; SK: string }>) => |
| 536 | use(({ table }) => |
| 537 | Effect.forEach( |
| 538 | Arr.chunksOf(keys, BATCH_WRITE_SIZE), |
| 539 | (batch) => |
| 540 | Effect.gen(function* () { |
| 541 | let requests: dynamodb.WriteRequest[] = batch.map((key) => ({ |
| 542 | DeleteRequest: { Key: { PK: S(key.PK), SK: S(key.SK) } }, |
| 543 | })); |
| 544 | yield* Effect.gen(function* () { |
| 545 | const result = yield* dynamodb.batchWriteItem({ |
| 546 | RequestItems: { [table]: requests }, |
| 547 | }); |
| 548 | requests = result.UnprocessedItems?.[table] ?? []; |
| 549 | }).pipe( |
| 550 | Effect.repeat({ |
| 551 | schedule: unprocessedBackoff, |
| 552 | until: () => requests.length === 0, |
| 553 | }), |
| 554 | ); |
| 555 | if (requests.length > 0) { |
| 556 | return yield* new StateStoreError({ |
| 557 | message: `DynamoDB left ${requests.length} state deletes unprocessed`, |
| 558 | }); |
| 559 | } |
| 560 | }), |
| 561 | { discard: true }, |
| 562 | ), |
| 563 | ); |
| 564 | |
| 565 | /** Every item of a partition, across pagination, strongly consistent. */ |
| 566 | const queryPartition = (pk: string, skPrefix?: string, projection?: string) => |
| 567 | use(({ table }) => |
| 568 | dynamodb.query |
| 569 | .pages({ |
| 570 | TableName: table, |
| 571 | ConsistentRead: true, |
| 572 | KeyConditionExpression: |
| 573 | skPrefix === undefined ? "PK = :pk" : "PK = :pk AND begins_with(SK, :sk)", |
| 574 | ExpressionAttributeValues: |
| 575 | skPrefix === undefined ? { ":pk": S(pk) } : { ":pk": S(pk), ":sk": S(skPrefix) }, |
| 576 | ...(projection === undefined |
| 577 | ? {} |
| 578 | : { |
| 579 | ProjectionExpression: projection, |
| 580 | ExpressionAttributeNames: { "#ref": "ref" }, |
| 581 | }), |
| 582 | }) |
| 583 | .pipe( |
| 584 | Stream.flatMap((page) => Stream.fromIterable(page.Items ?? [])), |
| 585 | Stream.runCollect, |
| 586 | Effect.map((items) => Array.from(items)), |
| 587 | ), |
| 588 | ); |
| 589 | |
| 590 | const heldError = (stack: string, stage: string) => |
| 591 | new StateStoreError({ |
| 592 | message: `${HELD_PREFIX} the ${label} state lock '${stack}/${stage}'`, |
| 593 | }); |
| 594 | |
| 595 | const lostError = (stack: string, stage: string) => |
| 596 | new StateStoreError({ |
| 597 | message: `the ${label} state lock '${stack}/${stage}' was lost mid-run; refusing to continue unlocked`, |
| 598 | }); |
| 599 | |
| 600 | const leaseKey = (stack: string, stage: string) => ({ |
| 601 | PK: S(stagePk(stack, stage)), |
| 602 | SK: S(LEASE), |
| 603 | }); |
| 604 | |
| 605 | const catalogKey = (stack: string, stage: string) => ({ |
| 606 | PK: S(CATALOG), |
| 607 | SK: S(catalogSk(stack, stage)), |
| 608 | }); |
| 609 | |
| 610 | /** Write the lease when `condition` holds; `false` when it doesn't. */ |
| 611 | const putLease = ( |
| 612 | stack: string, |
| 613 | stage: string, |
| 614 | token: string, |
| 615 | expiresAt: number, |
| 616 | condition: { |
| 617 | ConditionExpression: string; |
| 618 | ExpressionAttributeNames: Record<string, string>; |
| 619 | ExpressionAttributeValues: Item; |
| 620 | }, |
| 621 | ) => |
| 622 | use(({ table }) => |
| 623 | dynamodb |
| 624 | .putItem({ |
| 625 | TableName: table, |
| 626 | Item: { ...leaseKey(stack, stage), token: S(token), expiresAt: N(expiresAt) }, |
| 627 | ...condition, |
| 628 | }) |
| 629 | .pipe( |
| 630 | Effect.as(true), |
| 631 | Effect.catchTag("ConditionalCheckFailedException", () => Effect.succeed(false)), |
| 632 | ), |
| 633 | ); |
| 634 | |
| 635 | /** Take the stage's lease, held until `scope` closes. */ |
| 636 | const acquireLease = Effect.fn(function* (stack: string, stage: string, scope: Scope.Scope) { |
| 637 | const token = yield* Effect.sync(() => crypto.randomUUID()); |
| 638 | const now = yield* Clock.currentTimeMillis; |
| 639 | // Take the lease when nobody holds it or the holder let it expire. |
| 640 | const acquired = yield* putLease(stack, stage, token, now + LEASE_TTL_MS, { |
| 641 | ConditionExpression: "attribute_not_exists(PK) OR #expiresAt <= :now", |
| 642 | ExpressionAttributeNames: { "#expiresAt": "expiresAt" }, |
| 643 | ExpressionAttributeValues: { ":now": N(now) }, |
| 644 | }); |
| 645 | if (!acquired) return yield* heldError(stack, stage); |
| 646 | |
| 647 | const cursor = { lost: false }; |
| 648 | const gate = Semaphore.makeUnsafe(1); |
| 649 | |
| 650 | const renew = Semaphore.withPermits( |
| 651 | gate, |
| 652 | 1, |
| 653 | )( |
| 654 | Effect.gen(function* () { |
| 655 | if (cursor.lost) return; |
| 656 | const renewedAt = yield* Clock.currentTimeMillis; |
| 657 | const renewed = yield* putLease(stack, stage, token, renewedAt + LEASE_TTL_MS, { |
| 658 | ConditionExpression: "#token = :token", |
| 659 | ExpressionAttributeNames: { "#token": "token" }, |
| 660 | ExpressionAttributeValues: { ":token": S(token) }, |
| 661 | }); |
| 662 | if (!renewed) cursor.lost = true; |
| 663 | }), |
| 664 | ).pipe( |
| 665 | Effect.retry({ |
| 666 | while: () => !cursor.lost, |
| 667 | schedule: Schedule.spaced("1 second"), |
| 668 | times: 3, |
| 669 | }), |
| 670 | Effect.catch(() => |
| 671 | Effect.sync(() => { |
| 672 | cursor.lost = true; |
| 673 | }), |
| 674 | ), |
| 675 | ); |
| 676 | |
| 677 | const verify = Semaphore.withPermits( |
| 678 | gate, |
| 679 | 1, |
| 680 | )( |
| 681 | Effect.gen(function* () { |
| 682 | if (cursor.lost) return yield* lostError(stack, stage); |
| 683 | const existing = yield* use(({ table }) => |
| 684 | dynamodb.getItem({ |
| 685 | TableName: table, |
| 686 | Key: leaseKey(stack, stage), |
| 687 | ConsistentRead: true, |
| 688 | }), |
| 689 | ); |
| 690 | if (str(existing.Item, "token") !== token) { |
| 691 | cursor.lost = true; |
| 692 | return yield* lostError(stack, stage); |
| 693 | } |
| 694 | const observedAt = yield* Clock.currentTimeMillis; |
| 695 | if (Number(existing.Item?.expiresAt?.N ?? 0) <= observedAt) { |
| 696 | // The lease lapsed without anyone taking it: a refresh queued |
| 697 | // behind a large deploy's writes missed its window. Take it back, |
| 698 | // on condition it is still ours, so a deploy that took it in the |
| 699 | // meantime wins and this one stops. |
| 700 | const reclaimed = yield* putLease(stack, stage, token, observedAt + LEASE_TTL_MS, { |
| 701 | ConditionExpression: "#token = :token", |
| 702 | ExpressionAttributeNames: { "#token": "token" }, |
| 703 | ExpressionAttributeValues: { ":token": S(token) }, |
| 704 | }); |
| 705 | if (!reclaimed) { |
| 706 | cursor.lost = true; |
| 707 | return yield* lostError(stack, stage); |
| 708 | } |
| 709 | } |
| 710 | }), |
| 711 | ); |
| 712 | |
| 713 | // Stop the refresh before releasing, so a refresh cannot land after |
| 714 | // the lease has been released. |
| 715 | yield* Scope.addFinalizer( |
| 716 | scope, |
| 717 | Semaphore.withPermits( |
| 718 | gate, |
| 719 | 1, |
| 720 | )( |
| 721 | Effect.gen(function* () { |
| 722 | cursor.lost = true; |
| 723 | yield* use(({ table }) => |
| 724 | dynamodb |
| 725 | .deleteItem({ |
| 726 | TableName: table, |
| 727 | Key: leaseKey(stack, stage), |
| 728 | ConditionExpression: "#token = :token", |
| 729 | ExpressionAttributeNames: { "#token": "token" }, |
| 730 | ExpressionAttributeValues: { ":token": S(token) }, |
| 731 | }) |
| 732 | .pipe(Effect.catchTag("ConditionalCheckFailedException", () => Effect.void)), |
| 733 | ); |
| 734 | }).pipe(Effect.ignore), |
| 735 | ), |
| 736 | ); |
| 737 | yield* renew.pipe( |
| 738 | Effect.delay(Duration.millis(Math.floor(LEASE_TTL_MS / 3))), |
| 739 | Effect.forever, |
| 740 | Effect.interruptible, |
| 741 | Effect.forkIn(scope), |
| 742 | ); |
| 743 | |
| 744 | // The conditional put just proved the lease, so the first check is free. |
| 745 | return { |
| 746 | token, |
| 747 | checkLive: amortizeCheck(verify, LEASE_CHECK_TTL_MS, Date.now()), |
| 748 | } satisfies HeldLease; |
| 749 | }); |
| 750 | |
| 751 | /** |
| 752 | * Write one catalog item in a transaction with a check that `lease` is |
| 753 | * still the stage's. A holder that stalled past its lease, and lost it to |
| 754 | * another deploy, can't move the stage in or out of the catalog. |
| 755 | */ |
| 756 | const writeUnderLease = ( |
| 757 | stack: string, |
| 758 | stage: string, |
| 759 | lease: HeldLease, |
| 760 | write: (table: string) => dynamodb.TransactWriteItem, |
| 761 | ) => |
| 762 | use(({ table }) => |
| 763 | dynamodb |
| 764 | .transactWriteItems({ |
| 765 | TransactItems: [ |
| 766 | { |
| 767 | ConditionCheck: { |
| 768 | TableName: table, |
| 769 | Key: leaseKey(stack, stage), |
| 770 | ConditionExpression: "#token = :token", |
| 771 | ExpressionAttributeNames: { "#token": "token" }, |
| 772 | ExpressionAttributeValues: { ":token": S(lease.token) }, |
| 773 | }, |
| 774 | }, |
| 775 | write(table), |
| 776 | ], |
| 777 | }) |
| 778 | .pipe( |
| 779 | // A refresh of the lease landed during the transaction. |
| 780 | Effect.retry({ |
| 781 | while: (error) => |
| 782 | error._tag === "TransactionCanceledException" && |
| 783 | cancellationCodes(error).includes("TransactionConflict"), |
| 784 | schedule: Schedule.exponential("100 millis"), |
| 785 | times: 5, |
| 786 | }), |
| 787 | Effect.catchTag("TransactionCanceledException", (error) => |
| 788 | Effect.fail( |
| 789 | cancellationCodes(error)[0] === "ConditionalCheckFailed" |
| 790 | ? lostError(stack, stage) |
| 791 | : error, |
| 792 | ), |
| 793 | ), |
| 794 | ), |
| 795 | ).pipe(Effect.asVoid); |
| 796 | |
| 797 | const migrateCommand = (stack: string, stage: string) => |
| 798 | `alchemy state migrate --stack ${stack} --stage ${stage}`; |
| 799 | |
| 800 | /** The stage's catalog status; `undefined` when DynamoDB doesn't know it. */ |
| 801 | const catalogStatus = (stack: string, stage: string) => |
| 802 | use(({ table }) => |
| 803 | dynamodb.getItem({ |
| 804 | TableName: table, |
| 805 | Key: catalogKey(stack, stage), |
| 806 | ConsistentRead: true, |
| 807 | }), |
| 808 | ).pipe(Effect.map((result) => statusOf(result.Item))); |
| 809 | |
| 810 | /** |
| 811 | * Refuse a stage the S3 store still has and DynamoDB doesn't own: its |
| 812 | * state has not moved, and serving an empty stage would redeploy it. |
| 813 | */ |
| 814 | const refuseUnmigrated = Effect.fn(function* (stack: string, stage: string) { |
| 815 | const status = yield* catalogStatus(stack, stage); |
| 816 | const { legacy } = yield* names; |
| 817 | if (legacy === undefined || ownsStage(status)) return status; |
| 818 | if (!(yield* use(() => hasLegacyStage(legacy, stack, stage)))) return status; |
| 819 | const command = migrateCommand(stack, stage); |
| 820 | return yield* new StateStoreError({ |
| 821 | message: `Stage '${stack}/${stage}' is still in the S3 state store (s3://${legacy.bucket}/${legacy.prefix}). Run '${command}' to move it to DynamoDB.`, |
| 822 | requires: { action: "upgrade", command }, |
| 823 | }); |
| 824 | }); |
| 825 | |
| 826 | /** Every item of a stage as contents, without the migration check. */ |
| 827 | const readStage = Effect.fn(function* (stack: string, stage: string) { |
| 828 | const items = yield* queryPartition(stagePk(stack, stage)); |
| 829 | const records = new Map<string, string>(); |
| 830 | let output: string | undefined; |
| 831 | yield* Effect.forEach( |
| 832 | items, |
| 833 | (item) => |
| 834 | Effect.gen(function* () { |
| 835 | const sk = str(item, "SK"); |
| 836 | if (sk === OUTPUT) output = yield* itemText(item); |
| 837 | else if (sk?.startsWith(RECORD)) { |
| 838 | const fqn = str(item, "fqn"); |
| 839 | if (fqn !== undefined) records.set(fqn, yield* itemText(item)); |
| 840 | } |
| 841 | }), |
| 842 | { concurrency: 16, discard: true }, |
| 843 | ); |
| 844 | return { records, output } satisfies StageContents; |
| 845 | }); |
| 846 | |
| 847 | /** Load every record of a stage into a snapshot keyed by FQN. */ |
| 848 | const loadStage = Effect.fn(function* (stack: string, stage: string) { |
| 849 | const [items, status] = yield* Effect.all( |
| 850 | [queryPartition(stagePk(stack, stage), RECORD), refuseUnmigrated(stack, stage)], |
| 851 | { concurrency: 2 }, |
| 852 | ); |
| 853 | const records = yield* Effect.forEach( |
| 854 | items, |
| 855 | (item) => |
| 856 | Effect.gen(function* () { |
| 857 | const fqn = str(item, "fqn"); |
| 858 | if (fqn === undefined) { |
| 859 | return yield* new StateStoreError({ |
| 860 | message: `DynamoDB state item ${str(item, "PK")}/${str(item, "SK")} has no fqn`, |
| 861 | }); |
| 862 | } |
| 863 | return [fqn, { text: yield* itemText(item) }] as const; |
| 864 | }), |
| 865 | { concurrency: 16 }, |
| 866 | ); |
| 867 | return { records: new Map<string, StoredRecord>(records), status } satisfies Snapshot; |
| 868 | }); |
| 869 | |
| 870 | // One lease per (stack, stage), taken on the first operation and held |
| 871 | // for the store's lifetime. The snapshot is cached apart from the lease |
| 872 | // so a failed load retries without re-taking the lease this store holds. |
| 873 | const leases = keyedCache<HeldLease>(); |
| 874 | const snapshots = keyedCache<Snapshot>(); |
| 875 | // Stages this store has entered in the catalog. |
| 876 | const registered = keyedCache<void>(); |
| 877 | |
| 878 | const guarded = <A>( |
| 879 | request: { stack: string; stage: string }, |
| 880 | op: (snapshot: Snapshot, lease: HeldLease) => Effect.Effect<A, StateStoreError>, |
| 881 | ): Effect.Effect<A, StateStoreError> => { |
| 882 | const key = `${request.stack}/${request.stage}`; |
| 883 | return leases.get(key, acquireLease(request.stack, request.stage, leaseScope)).pipe( |
| 884 | Effect.tap((lease) => lease.checkLive), |
| 885 | Effect.flatMap((lease) => |
| 886 | snapshots |
| 887 | .get(key, loadStage(request.stack, request.stage)) |
| 888 | .pipe(Effect.flatMap((snapshot) => op(snapshot, lease))), |
| 889 | ), |
| 890 | ); |
| 891 | }; |
| 892 | |
| 893 | const verifyHeldLeases: Effect.Effect<void, StateStoreError> = Effect.suspend(() => |
| 894 | Effect.forEach( |
| 895 | Array.from(leases.entries.values()), |
| 896 | (lease) => lease.pipe(Effect.flatMap((held) => held.checkLive)), |
| 897 | { discard: true }, |
| 898 | ), |
| 899 | ); |
| 900 | |
| 901 | /** |
| 902 | * Enter the stage in the catalog `listStacks` and `listStages` read, once |
| 903 | * per stage per store. |
| 904 | */ |
| 905 | const register = (stack: string, stage: string, snapshot: Snapshot, lease: HeldLease) => |
| 906 | registered.get( |
| 907 | `${stack}/${stage}`, |
| 908 | Effect.gen(function* () { |
| 909 | const { legacy, table } = yield* names; |
| 910 | // A stage new to DynamoDB is fenced in the S3 store too, so a |
| 911 | // teammate still on the S3 store fails at its lock instead of |
| 912 | // deploying the stage a second time. |
| 913 | if ( |
| 914 | legacy !== undefined && |
| 915 | (snapshot.status === undefined || snapshot.status === "rolled-back") |
| 916 | ) { |
| 917 | const fence = yield* use(() => fenceLegacyStage(legacy, stack, stage, table)); |
| 918 | if (!fence.fenced) { |
| 919 | return yield* new StateStoreError({ |
| 920 | message: `${HELD_PREFIX} the S3 state lock '${stack}/${stage}': a deploy on the S3 state store is creating this stage`, |
| 921 | }); |
| 922 | } |
| 923 | } |
| 924 | // Keep the entry's migration record; a destroyed stage is live again. |
| 925 | yield* writeUnderLease(stack, stage, lease, (table) => ({ |
| 926 | Update: { |
| 927 | TableName: table, |
| 928 | Key: catalogKey(stack, stage), |
| 929 | UpdateExpression: "SET #stack = :stack, #stage = :stage REMOVE #status", |
| 930 | ExpressionAttributeNames: { "#stack": "stack", "#stage": "stage", "#status": "status" }, |
| 931 | ExpressionAttributeValues: { ":stack": S(stack), ":stage": S(stage) }, |
| 932 | }, |
| 933 | })); |
| 934 | snapshot.status = "active"; |
| 935 | }), |
| 936 | ); |
| 937 | |
| 938 | /** Every cataloged stage, of one stack or all of them, in one query. */ |
| 939 | const catalog = (stack?: string) => |
| 940 | queryPartition(CATALOG, stack === undefined ? "STAGE#" : catalogPrefix(stack)).pipe( |
| 941 | Effect.map((items) => |
| 942 | items.flatMap((item) => { |
| 943 | const stack = str(item, "stack"); |
| 944 | const stage = str(item, "stage"); |
| 945 | const status = statusOf(item); |
| 946 | return stack === undefined || stage === undefined || status === undefined |
| 947 | ? [] |
| 948 | : [{ stack, stage, status }]; |
| 949 | }), |
| 950 | ), |
| 951 | ); |
| 952 | |
| 953 | const stageKey = (stack: string, stage: string) => `${stack}\u0000${stage}`; |
| 954 | |
| 955 | /** Stages of a stack, in DynamoDB or still in the S3 store. */ |
| 956 | const listStages = (stack: string) => |
| 957 | Effect.all( |
| 958 | [ |
| 959 | catalog(stack), |
| 960 | names.pipe( |
| 961 | Effect.flatMap(({ legacy }) => |
| 962 | legacy === undefined ? Effect.succeed([]) : use(() => listLegacyStages(legacy, stack)), |
| 963 | ), |
| 964 | ), |
| 965 | ], |
| 966 | { concurrency: 2 }, |
| 967 | ).pipe( |
| 968 | Effect.map(([entries, legacy]) => { |
| 969 | const owned = new Set(entries.filter((e) => ownsStage(e.status)).map((e) => e.stage)); |
| 970 | return Array.from( |
| 971 | new Set([ |
| 972 | ...entries.filter((entry) => entry.status === "active").map((entry) => entry.stage), |
| 973 | ...legacy.filter((stage) => !owned.has(stage)), |
| 974 | ]), |
| 975 | ); |
| 976 | }), |
| 977 | ); |
| 978 | |
| 979 | /** Delete a stage's records, output and overflow objects. */ |
| 980 | const deleteStageItems = Effect.fn(function* (stack: string, stage: string) { |
| 981 | const items = (yield* queryPartition(stagePk(stack, stage), undefined, "PK, SK, #ref")).filter( |
| 982 | // Leave the lease. Releasing it here would open the stage before |
| 983 | // this delete finishes, and the store still holds the lock. |
| 984 | (item) => str(item, "SK") !== LEASE, |
| 985 | ); |
| 986 | yield* deleteItems(items.map((item) => ({ PK: str(item, "PK")!, SK: str(item, "SK")! }))); |
| 987 | // Overflow objects of the stage, including any a crashed write |
| 988 | // uploaded but never committed. |
| 989 | yield* use(({ bucket }) => |
| 990 | Effect.gen(function* () { |
| 991 | if (!(yield* stateBucketExists(bucket))) return; |
| 992 | const objects = yield* s3.listObjectsV2 |
| 993 | .pages({ Bucket: bucket, Prefix: blobPrefix(stack, stage) }) |
| 994 | .pipe( |
| 995 | Stream.flatMap((page) => Stream.fromIterable(page.Contents ?? [])), |
| 996 | Stream.map((object) => object.Key), |
| 997 | Stream.filter((key): key is string => key !== undefined), |
| 998 | Stream.runCollect, |
| 999 | Effect.map((keys) => Array.from(keys)), |
| 1000 | ); |
| 1001 | yield* Effect.forEach( |
| 1002 | Arr.chunksOf(objects, S3_DELETE_BATCH_SIZE), |
| 1003 | (batch) => |
| 1004 | s3.deleteObjects({ |
| 1005 | Bucket: bucket, |
| 1006 | Delete: { Objects: batch.map((Key) => ({ Key })), Quiet: true }, |
| 1007 | }), |
| 1008 | { discard: true }, |
| 1009 | ); |
| 1010 | }), |
| 1011 | ); |
| 1012 | }); |
| 1013 | |
| 1014 | /** Delete a stage. Its catalog entry stays as a tombstone while an S3 copy may exist. */ |
| 1015 | const deleteStage = (stack: string, stage: string) => |
| 1016 | guarded({ stack, stage }, (snapshot, lease) => |
| 1017 | Effect.gen(function* () { |
| 1018 | yield* deleteStageItems(stack, stage); |
| 1019 | snapshot.records.clear(); |
| 1020 | if (snapshot.status !== undefined) { |
| 1021 | const { legacy } = yield* names; |
| 1022 | // With an S3 store behind it, a deleted entry would read as a |
| 1023 | // stage that never moved, and bring its pre-migration copy back. |
| 1024 | yield* writeUnderLease(stack, stage, lease, (table) => |
| 1025 | legacy === undefined |
| 1026 | ? { Delete: { TableName: table, Key: catalogKey(stack, stage) } } |
| 1027 | : { |
| 1028 | Update: { |
| 1029 | TableName: table, |
| 1030 | Key: catalogKey(stack, stage), |
| 1031 | UpdateExpression: "SET #stack = :stack, #stage = :stage, #status = :destroyed", |
| 1032 | ExpressionAttributeNames: { |
| 1033 | "#stack": "stack", |
| 1034 | "#stage": "stage", |
| 1035 | "#status": "status", |
| 1036 | }, |
| 1037 | ExpressionAttributeValues: { |
| 1038 | ":stack": S(stack), |
| 1039 | ":stage": S(stage), |
| 1040 | ":destroyed": S("destroyed"), |
| 1041 | }, |
| 1042 | }, |
| 1043 | }, |
| 1044 | ); |
| 1045 | snapshot.status = legacy === undefined ? undefined : "destroyed"; |
| 1046 | } |
| 1047 | // A later write to the stage enters it in the catalog again. |
| 1048 | registered.entries.delete(`${stack}/${stage}`); |
| 1049 | }), |
| 1050 | ); |
| 1051 | |
| 1052 | const state: StateService = { |
| 1053 | id: "dynamodb", |
| 1054 | getVersion: () => Effect.succeed(STATE_STORE_VERSION), |
| 1055 | // A stack is listed while it has a live stage in DynamoDB, or a stage |
| 1056 | // the S3 store still has. |
| 1057 | listStacks: () => |
| 1058 | verifyHeldLeases.pipe( |
| 1059 | Effect.andThen(Effect.all([catalog(), names], { concurrency: 2 })), |
| 1060 | Effect.flatMap(([entries, { legacy }]) => { |
| 1061 | const owned = new Set( |
| 1062 | entries.filter((e) => ownsStage(e.status)).map((e) => stageKey(e.stack, e.stage)), |
| 1063 | ); |
| 1064 | const live = entries.filter((entry) => entry.status === "active").map((e) => e.stack); |
| 1065 | if (legacy === undefined) return Effect.succeed(Array.from(new Set(live))); |
| 1066 | return use(() => |
| 1067 | listLegacyStacks(legacy).pipe( |
| 1068 | Effect.flatMap((stacks) => |
| 1069 | Effect.forEach( |
| 1070 | stacks, |
| 1071 | (stack) => |
| 1072 | listLegacyStages(legacy, stack).pipe( |
| 1073 | Effect.map((stages) => |
| 1074 | stages.some((stage) => !owned.has(stageKey(stack, stage))) ? [stack] : [], |
| 1075 | ), |
| 1076 | ), |
| 1077 | { concurrency: 8 }, |
| 1078 | ), |
| 1079 | ), |
| 1080 | ), |
| 1081 | ).pipe( |
| 1082 | Effect.map((legacyStacks) => Array.from(new Set([...live, ...legacyStacks.flat()]))), |
| 1083 | ); |
| 1084 | }), |
| 1085 | ), |
| 1086 | listStages: (stack) => verifyHeldLeases.pipe(Effect.andThen(listStages(stack))), |
| 1087 | get: (request) => |
| 1088 | guarded(request, ({ records }) => { |
| 1089 | const record = records.get(request.fqn); |
| 1090 | return record === undefined |
| 1091 | ? Effect.undefined |
| 1092 | : parse<PersistedState>(record.text, `record '${request.fqn}'`); |
| 1093 | }), |
| 1094 | getReplacedResources: (request) => |
| 1095 | guarded(request, ({ records }) => |
| 1096 | Effect.forEach(Array.from(records), ([fqn, record]) => |
| 1097 | parse<PersistedState>(record.text, `record '${fqn}'`), |
| 1098 | ).pipe(Effect.map((values) => values.filter((value) => value.status === "replaced"))), |
| 1099 | ) as ReturnType<StateService["getReplacedResources"]>, |
| 1100 | set: (request) => |
| 1101 | guarded(request, (snapshot, lease) => |
| 1102 | Effect.gen(function* () { |
| 1103 | yield* register(request.stack, request.stage, snapshot, lease); |
| 1104 | const text = JSON.stringify(encodeState(request.value)); |
| 1105 | yield* putPayload( |
| 1106 | { |
| 1107 | PK: stagePk(request.stack, request.stage), |
| 1108 | SK: yield* recordSk(request.fqn), |
| 1109 | stack: request.stack, |
| 1110 | stage: request.stage, |
| 1111 | }, |
| 1112 | text, |
| 1113 | { fqn: S(request.fqn) }, |
| 1114 | ); |
| 1115 | snapshot.records.set(request.fqn, { text }); |
| 1116 | return request.value; |
| 1117 | }), |
| 1118 | ), |
| 1119 | delete: (request) => |
| 1120 | guarded(request, ({ records }) => |
| 1121 | Effect.gen(function* () { |
| 1122 | const previous = yield* use(({ table }) => |
| 1123 | Effect.flatMap(recordSk(request.fqn), (sk) => |
| 1124 | dynamodb.deleteItem({ |
| 1125 | TableName: table, |
| 1126 | Key: { PK: S(stagePk(request.stack, request.stage)), SK: S(sk) }, |
| 1127 | ReturnValues: "ALL_OLD", |
| 1128 | }), |
| 1129 | ), |
| 1130 | ); |
| 1131 | records.delete(request.fqn); |
| 1132 | yield* deleteOverflow(str(previous.Attributes, "ref")); |
| 1133 | }), |
| 1134 | ), |
| 1135 | // A stack-wide delete locks every stage it is about to remove, so a |
| 1136 | // concurrent deploy fails the lease instead of racing the delete. |
| 1137 | deleteStack: ({ stack, stage }) => |
| 1138 | stage === undefined |
| 1139 | ? verifyHeldLeases.pipe( |
| 1140 | Effect.andThen(listStages(stack)), |
| 1141 | Effect.flatMap((stages) => |
| 1142 | Effect.forEach(stages, (found) => deleteStage(stack, found), { discard: true }), |
| 1143 | ), |
| 1144 | ) |
| 1145 | : deleteStage(stack, stage), |
| 1146 | list: (request) => |
| 1147 | guarded(request, ({ records }) => Effect.succeed(Array.from(records.keys()))), |
| 1148 | // Always read fresh, without the lease: cross-stack references read |
| 1149 | // the outputs of stages another deploy may hold. |
| 1150 | getOutput: Effect.fn(function* (request) { |
| 1151 | const result = yield* use(({ table }) => |
| 1152 | dynamodb.getItem({ |
| 1153 | TableName: table, |
| 1154 | Key: { PK: S(stagePk(request.stack, request.stage)), SK: S(OUTPUT) }, |
| 1155 | ConsistentRead: true, |
| 1156 | }), |
| 1157 | ); |
| 1158 | if (result.Item !== undefined) { |
| 1159 | return yield* parse<unknown>(yield* itemText(result.Item), "stack output"); |
| 1160 | } |
| 1161 | // A stage that hasn't moved yet keeps its output in the S3 store. |
| 1162 | // Reading it takes no lease, so a reference to it still resolves. |
| 1163 | const { legacy } = yield* names; |
| 1164 | if (legacy === undefined) return undefined; |
| 1165 | if (ownsStage(yield* catalogStatus(request.stack, request.stage))) return undefined; |
| 1166 | return yield* use(() => readLegacyOutput(legacy, request.stack, request.stage)); |
| 1167 | }), |
| 1168 | setOutput: (request) => |
| 1169 | guarded(request, (snapshot, lease) => |
| 1170 | Effect.gen(function* () { |
| 1171 | if (request.value === undefined) { |
| 1172 | const previous = yield* use(({ table }) => |
| 1173 | dynamodb.deleteItem({ |
| 1174 | TableName: table, |
| 1175 | Key: { PK: S(stagePk(request.stack, request.stage)), SK: S(OUTPUT) }, |
| 1176 | ReturnValues: "ALL_OLD", |
| 1177 | }), |
| 1178 | ); |
| 1179 | yield* deleteOverflow(str(previous.Attributes, "ref")); |
| 1180 | return request.value; |
| 1181 | } |
| 1182 | yield* register(request.stack, request.stage, snapshot, lease); |
| 1183 | yield* putPayload( |
| 1184 | { |
| 1185 | PK: stagePk(request.stack, request.stage), |
| 1186 | SK: OUTPUT, |
| 1187 | stack: request.stack, |
| 1188 | stage: request.stage, |
| 1189 | }, |
| 1190 | JSON.stringify(encodeState(request.value)), |
| 1191 | {}, |
| 1192 | ); |
| 1193 | return request.value; |
| 1194 | }), |
| 1195 | ), |
| 1196 | }; |
| 1197 | |
| 1198 | /** |
| 1199 | * Run `body` holding the stage's lease, released as soon as it finishes |
| 1200 | * rather than when the store closes. |
| 1201 | */ |
| 1202 | const exclusive = <A, R>( |
| 1203 | stack: string, |
| 1204 | stage: string, |
| 1205 | body: (lease: HeldLease) => Effect.Effect<A, StateStoreError, R>, |
| 1206 | ) => |
| 1207 | Effect.scoped( |
| 1208 | Effect.gen(function* () { |
| 1209 | const key = `${stack}/${stage}`; |
| 1210 | const scope = yield* Effect.scope; |
| 1211 | const lease = yield* acquireLease(stack, stage, scope); |
| 1212 | yield* Scope.addFinalizer( |
| 1213 | scope, |
| 1214 | Effect.sync(() => { |
| 1215 | snapshots.entries.delete(key); |
| 1216 | registered.entries.delete(key); |
| 1217 | }), |
| 1218 | ); |
| 1219 | return yield* body(lease); |
| 1220 | }), |
| 1221 | ); |
| 1222 | |
| 1223 | /** |
| 1224 | * Make the stage hold exactly `contents`, check it does, and enter it in |
| 1225 | * the catalog: the commit that makes DynamoDB the stage's store. |
| 1226 | */ |
| 1227 | const commitStage = Effect.fn(function* ( |
| 1228 | stack: string, |
| 1229 | stage: string, |
| 1230 | lease: HeldLease, |
| 1231 | contents: StageContents, |
| 1232 | migratedFrom: string, |
| 1233 | beforeCommit: Effect.Effect<void, StateStoreError> = Effect.void, |
| 1234 | ) { |
| 1235 | yield* lease.checkLive; |
| 1236 | const pk = stagePk(stack, stage); |
| 1237 | const wanted = new Set(yield* Effect.forEach(Array.from(contents.records.keys()), recordSk)); |
| 1238 | if (contents.output !== undefined) wanted.add(OUTPUT); |
| 1239 | // Items an earlier, interrupted copy left that the source no longer has. |
| 1240 | const stale = (yield* queryPartition(pk, undefined, "PK, SK, #ref")).filter((item) => { |
| 1241 | const sk = str(item, "SK"); |
| 1242 | return sk !== LEASE && sk !== undefined && !wanted.has(sk); |
| 1243 | }); |
| 1244 | yield* deleteItems(stale.map((item) => ({ PK: str(item, "PK")!, SK: str(item, "SK")! }))); |
| 1245 | yield* Effect.forEach(stale, (item) => deleteOverflow(str(item, "ref")), { discard: true }); |
| 1246 | yield* Effect.forEach( |
| 1247 | Array.from(contents.records), |
| 1248 | ([fqn, text]) => |
| 1249 | Effect.flatMap(recordSk(fqn), (sk) => |
| 1250 | putPayload({ PK: pk, SK: sk, stack, stage }, text, { fqn: S(fqn) }), |
| 1251 | ), |
| 1252 | { concurrency: 16, discard: true }, |
| 1253 | ); |
| 1254 | if (contents.output !== undefined) { |
| 1255 | yield* putPayload({ PK: pk, SK: OUTPUT, stack, stage }, contents.output, {}); |
| 1256 | } |
| 1257 | // Read back what landed before it becomes the stage's state. |
| 1258 | const copied = yield* readStage(stack, stage); |
| 1259 | const mismatched = [ |
| 1260 | ...Array.from(contents.records).filter(([fqn, text]) => copied.records.get(fqn) !== text), |
| 1261 | ...Array.from(copied.records.keys()).filter((fqn) => !contents.records.has(fqn)), |
| 1262 | ]; |
| 1263 | if (mismatched.length > 0 || copied.output !== contents.output) { |
| 1264 | return yield* new StateStoreError({ |
| 1265 | message: `DynamoDB copy of '${stack}/${stage}' does not match its source (${mismatched.length} records differ${copied.output !== contents.output ? ", output differs" : ""})`, |
| 1266 | }); |
| 1267 | } |
| 1268 | yield* beforeCommit; |
| 1269 | const migratedAt = new Date(yield* Clock.currentTimeMillis).toISOString(); |
| 1270 | yield* writeUnderLease(stack, stage, lease, (table) => ({ |
| 1271 | Put: { |
| 1272 | TableName: table, |
| 1273 | Item: { |
| 1274 | ...catalogKey(stack, stage), |
| 1275 | stack: S(stack), |
| 1276 | stage: S(stage), |
| 1277 | migratedFrom: S(migratedFrom), |
| 1278 | migratedAt: S(migratedAt), |
| 1279 | }, |
| 1280 | }, |
| 1281 | })); |
| 1282 | }); |
| 1283 | |
| 1284 | /** Mark the stage as moved back: the S3 store is its store again. */ |
| 1285 | const markRolledBack = (stack: string, stage: string, lease: HeldLease) => |
| 1286 | writeUnderLease(stack, stage, lease, (table) => ({ |
| 1287 | Update: { |
| 1288 | TableName: table, |
| 1289 | Key: catalogKey(stack, stage), |
| 1290 | UpdateExpression: "SET #status = :status", |
| 1291 | ExpressionAttributeNames: { "#status": "status" }, |
| 1292 | ExpressionAttributeValues: { ":status": S("rolled-back") }, |
| 1293 | }, |
| 1294 | })); |
| 1295 | |
| 1296 | const service: DynamoDBStateService = { |
| 1297 | ...state, |
| 1298 | migrate: (request) => |
| 1299 | (request.rollback |
| 1300 | ? rollbackToS3(service, request) |
| 1301 | : migrateToDynamoDB(service, request) |
| 1302 | ).pipe(Effect.provideContext(context)), |
| 1303 | migration: { |
| 1304 | names, |
| 1305 | provision: names.pipe( |
| 1306 | Effect.tap((resolved) => run(provision(resolved.table, resolved.bucket, resolved.region))), |
| 1307 | ), |
| 1308 | catalog: () => catalog(), |
| 1309 | status: catalogStatus, |
| 1310 | exclusive, |
| 1311 | readStage, |
| 1312 | commitStage, |
| 1313 | markRolledBack, |
| 1314 | deleteStageItems: (stack, stage, lease) => |
| 1315 | lease.checkLive.pipe(Effect.andThen(deleteStageItems(stack, stage))), |
| 1316 | }, |
| 1317 | }; |
| 1318 | return service; |
| 1319 | }); |
| 1320 | |
| 1321 | /** Why each item of a cancelled transaction failed, in request order. */ |
| 1322 | const cancellationCodes = (error: dynamodb.TransactionCanceledException) => |
| 1323 | // The generated schema leaves the reason list untyped. |
| 1324 | ((error.CancellationReasons ?? []) as ReadonlyArray<dynamodb.CancellationReason>).map( |
| 1325 | (reason) => reason.Code, |
| 1326 | ); |
| 1327 | |
| 1328 | /** A stage's snapshot: its records and its catalog status when loaded. */ |
| 1329 | interface Snapshot { |
| 1330 | readonly records: Map<string, StoredRecord>; |
| 1331 | status: CatalogStatus | undefined; |
| 1332 | } |
| 1333 | |
| 1334 | const statusOf = (item: Item | undefined): CatalogStatus | undefined => |
| 1335 | item === undefined ? undefined : ((str(item, "status") as CatalogStatus | undefined) ?? "active"); |
| 1336 | |
| 1337 | /** |
| 1338 | * The DynamoDB store, with the operations `alchemy state migrate` moves |
| 1339 | * stages with. |
| 1340 | */ |
| 1341 | export interface DynamoDBStateService extends StateService { |
| 1342 | readonly migration: { |
| 1343 | /** The table, overflow bucket and S3 store this store addresses. */ |
| 1344 | readonly names: Effect.Effect< |
| 1345 | { table: string; bucket: string; region: string; legacy: LegacyLocation | undefined }, |
| 1346 | StateStoreError |
| 1347 | >; |
| 1348 | /** {@link names}, after creating the table and overflow bucket if missing. */ |
| 1349 | readonly provision: Effect.Effect< |
| 1350 | { table: string; bucket: string; region: string; legacy: LegacyLocation | undefined }, |
| 1351 | StateStoreError |
| 1352 | >; |
| 1353 | readonly catalog: () => Effect.Effect< |
| 1354 | ReadonlyArray<{ stack: string; stage: string; status: CatalogStatus }>, |
| 1355 | StateStoreError |
| 1356 | >; |
| 1357 | readonly status: ( |
| 1358 | stack: string, |
| 1359 | stage: string, |
| 1360 | ) => Effect.Effect<CatalogStatus | undefined, StateStoreError>; |
| 1361 | readonly exclusive: <A, R>( |
| 1362 | stack: string, |
| 1363 | stage: string, |
| 1364 | body: (lease: HeldLease) => Effect.Effect<A, StateStoreError, R>, |
| 1365 | ) => Effect.Effect<A, StateStoreError, Exclude<R, Scope.Scope>>; |
| 1366 | readonly readStage: ( |
| 1367 | stack: string, |
| 1368 | stage: string, |
| 1369 | ) => Effect.Effect<StageContents, StateStoreError>; |
| 1370 | readonly commitStage: ( |
| 1371 | stack: string, |
| 1372 | stage: string, |
| 1373 | lease: HeldLease, |
| 1374 | contents: StageContents, |
| 1375 | migratedFrom: string, |
| 1376 | beforeCommit?: Effect.Effect<void, StateStoreError>, |
| 1377 | ) => Effect.Effect<void, StateStoreError>; |
| 1378 | readonly markRolledBack: ( |
| 1379 | stack: string, |
| 1380 | stage: string, |
| 1381 | lease: HeldLease, |
| 1382 | ) => Effect.Effect<void, StateStoreError>; |
| 1383 | readonly deleteStageItems: ( |
| 1384 | stack: string, |
| 1385 | stage: string, |
| 1386 | lease: HeldLease, |
| 1387 | ) => Effect.Effect<void, StateStoreError>; |
| 1388 | }; |
| 1389 | } |
| 1390 | |
| 1391 | /** The table's description, or `undefined` when it doesn't exist. */ |
| 1392 | const describeStateTable = (table: string) => |
| 1393 | dynamodb.describeTable({ TableName: table }).pipe( |
| 1394 | Effect.map((result) => result.Table), |
| 1395 | Effect.catchTag("ResourceNotFoundException", () => Effect.void), |
| 1396 | ); |
| 1397 | |
| 1398 | const validateKeySchema = (table: dynamodb.TableDescription) => { |
| 1399 | const keys = table.KeySchema ?? []; |
| 1400 | const hash = keys.find((key) => key.KeyType === "HASH")?.AttributeName; |
| 1401 | const range = keys.find((key) => key.KeyType === "RANGE")?.AttributeName; |
| 1402 | return hash === "PK" && range === "SK" |
| 1403 | ? Effect.void |
| 1404 | : Effect.fail( |
| 1405 | new StateStoreError({ |
| 1406 | message: `DynamoDB state table '${table.TableName}' must have partition key PK and sort key SK (found ${hash ?? "none"} and ${range ?? "none"})`, |
| 1407 | }), |
| 1408 | ); |
| 1409 | }; |
| 1410 | |
| 1411 | class TableNotActive extends Data.TaggedError("TableNotActive") {} |
| 1412 | |
| 1413 | const waitForTableActive = (table: string) => |
| 1414 | dynamodb.describeTable({ TableName: table }).pipe( |
| 1415 | Effect.flatMap((result) => |
| 1416 | result.Table?.TableStatus === "ACTIVE" |
| 1417 | ? Effect.succeed(result.Table) |
| 1418 | : Effect.fail(new TableNotActive()), |
| 1419 | ), |
| 1420 | Effect.retry({ |
| 1421 | // A just-created table can briefly describe as missing. |
| 1422 | while: (error) => |
| 1423 | error._tag === "TableNotActive" || error._tag === "ResourceNotFoundException", |
| 1424 | schedule: Schedule.spaced("1 second"), |
| 1425 | times: 60, |
| 1426 | }), |
| 1427 | Effect.catchTag("TableNotActive", () => |
| 1428 | Effect.fail( |
| 1429 | new StateStoreError({ message: `DynamoDB state table '${table}' never became ACTIVE` }), |
| 1430 | ), |
| 1431 | ), |
| 1432 | ); |
| 1433 | |
| 1434 | /** |
| 1435 | * Observe-then-ensure the state table: create it if missing (tolerating |
| 1436 | * create races), with deletion protection and, given `kmsKeyId`, that KMS |
| 1437 | * key; wait for it to become ACTIVE, and enable point-in-time recovery. |
| 1438 | */ |
| 1439 | const ensureStateTable = Effect.fn(function* (table: string, kmsKeyId?: string) { |
| 1440 | const observed = yield* describeStateTable(table); |
| 1441 | if (observed === undefined) { |
| 1442 | yield* Effect.logInfo(`DynamoDB state store: creating table ${table}`); |
| 1443 | yield* dynamodb |
| 1444 | .createTable({ |
| 1445 | TableName: table, |
| 1446 | AttributeDefinitions: [ |
| 1447 | { AttributeName: "PK", AttributeType: "S" }, |
| 1448 | { AttributeName: "SK", AttributeType: "S" }, |
| 1449 | ], |
| 1450 | KeySchema: [ |
| 1451 | { AttributeName: "PK", KeyType: "HASH" }, |
| 1452 | { AttributeName: "SK", KeyType: "RANGE" }, |
| 1453 | ], |
| 1454 | BillingMode: "PAY_PER_REQUEST", |
| 1455 | // The table holds the state of every stack on the account. |
| 1456 | DeletionProtectionEnabled: true, |
| 1457 | ...(kmsKeyId === undefined |
| 1458 | ? {} |
| 1459 | : { SSESpecification: { Enabled: true, SSEType: "KMS", KMSMasterKeyId: kmsKeyId } }), |
| 1460 | }) |
| 1461 | // Another first caller is creating it. |
| 1462 | .pipe(Effect.catchTag("ResourceInUseException", () => Effect.void)); |
| 1463 | } else { |
| 1464 | yield* validateKeySchema(observed); |
| 1465 | } |
| 1466 | yield* waitForTableActive(table); |
| 1467 | |
| 1468 | const backups = yield* dynamodb.describeContinuousBackups({ TableName: table }); |
| 1469 | if ( |
| 1470 | backups.ContinuousBackupsDescription?.PointInTimeRecoveryDescription |
| 1471 | ?.PointInTimeRecoveryStatus !== "ENABLED" |
| 1472 | ) { |
| 1473 | yield* dynamodb |
| 1474 | .updateContinuousBackups({ |
| 1475 | TableName: table, |
| 1476 | PointInTimeRecoverySpecification: { PointInTimeRecoveryEnabled: true }, |
| 1477 | }) |
| 1478 | .pipe( |
| 1479 | // Backups become available shortly after the table is ACTIVE. |
| 1480 | Effect.retry({ |
| 1481 | while: (error) => error._tag === "ContinuousBackupsUnavailableException", |
| 1482 | schedule: Schedule.spaced("2 seconds"), |
| 1483 | times: 15, |
| 1484 | }), |
| 1485 | ); |
| 1486 | } |
| 1487 | }); |
| 1488 | |
| 1489 | /** |
| 1490 | * Create the state table, or converge an existing one: what first use of |
| 1491 | * {@link dynamoDBState} does. |
| 1492 | */ |
| 1493 | export const provisionStateTable = ( |
| 1494 | table: string, |
| 1495 | kmsKeyId?: string, |
| 1496 | ): Effect.Effect<void, StateStoreError, AwsDeps> => |
| 1497 | ensureStateTable(table, kmsKeyId).pipe(Effect.mapError(toError)); |
| 1498 | |
| 1499 | /** Whether a state table exists. */ |
| 1500 | export const stateTableExists = (table: string) => |
| 1501 | describeStateTable(table).pipe(Effect.map((observed) => observed !== undefined)); |