The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 04:45 UTC
| 1 | import type { StateMigration, StateMigrationRequest } from "alchemy/State/State"; |
| 2 | import { StateStoreError } from "alchemy/State/State"; |
| 3 | import * as Effect from "effect/Effect"; |
| 4 | import type { DynamoDBStateService, HeldLease } from "./DynamoDBState.ts"; |
| 5 | import { |
| 6 | fenceLegacyStage, |
| 7 | isLeaseHeld, |
| 8 | type LegacyLocation, |
| 9 | listLegacyStacks, |
| 10 | listLegacyStages, |
| 11 | readLegacyStage, |
| 12 | setStageAccess, |
| 13 | unfenceLegacyStage, |
| 14 | writeLegacyStage, |
| 15 | } from "./LegacyLayout.ts"; |
| 16 | |
| 17 | /** |
| 18 | * Moving stages from the S3 state store to the DynamoDB one, and back. |
| 19 | * |
| 20 | * Each stage moves on its own, under both stores' leases: |
| 21 | * |
| 22 | * 1. Hold the stage's DynamoDB lease. A stage DynamoDB owns has moved. |
| 23 | * 2. Fence the stage in S3: replace its lease with one that never expires, |
| 24 | * which every S3 store since 2.0.0-beta.84 reads as held, and deny |
| 25 | * writes to its objects in the bucket policy, which stops the versions |
| 26 | * before it. A deploy that holds the lease keeps it, and the stage is |
| 27 | * reported as blocked. |
| 28 | * 3. Copy every record and the output, and delete what an earlier, |
| 29 | * interrupted copy left that the source no longer has. |
| 30 | * 4. Read the copy back and compare it with the source. |
| 31 | * 5. Enter the stage in the DynamoDB catalog, on condition the lease is |
| 32 | * still ours. This write is the switch: the DynamoDB store refuses a |
| 33 | * stage the S3 store has until it is there. |
| 34 | * 6. Deny reads of the stage's objects too, so any older version fails at |
| 35 | * its first read instead of deploying from the copy left behind. S3 can |
| 36 | * take minutes to enforce this second policy update everywhere; writes |
| 37 | * stay denied throughout, so an older version that reads the copy in |
| 38 | * that window still can't record anything. |
| 39 | * |
| 40 | * A failure at any step leaves the S3 copy authoritative and, once fenced, |
| 41 | * unchanged, so running the migration again converges. The S3 objects |
| 42 | * are kept. |
| 43 | */ |
| 44 | |
| 45 | /** |
| 46 | * A stage's entry in the DynamoDB catalog: |
| 47 | * |
| 48 | * - `active`: DynamoDB holds the stage. |
| 49 | * - `destroyed`: the stage was destroyed in DynamoDB. The entry stays, so an |
| 50 | * S3 copy from before the migration doesn't come back. |
| 51 | * - `rolled-back`: moved back; the S3 store holds the stage again. |
| 52 | */ |
| 53 | export type CatalogStatus = "active" | "destroyed" | "rolled-back"; |
| 54 | |
| 55 | /** Whether DynamoDB is the stage's store, whatever the S3 store still has. */ |
| 56 | export const ownsStage = (status: CatalogStatus | undefined) => |
| 57 | status === "active" || status === "destroyed"; |
| 58 | |
| 59 | export type StageMigrationStatus = |
| 60 | /** Moved by this run. */ |
| 61 | | "migrated" |
| 62 | /** Moved by an earlier run, or never in the S3 store. */ |
| 63 | | "already-migrated" |
| 64 | /** Moved back to the S3 store by this run. */ |
| 65 | | "rolled-back" |
| 66 | /** Not in DynamoDB, so there is nothing to roll back. */ |
| 67 | | "not-migrated" |
| 68 | /** Another deploy holds the stage. */ |
| 69 | | "blocked" |
| 70 | /** Moving the stage failed; the reason says why. */ |
| 71 | | "failed"; |
| 72 | |
| 73 | export interface StageMigration extends StateMigration { |
| 74 | readonly status: StageMigrationStatus; |
| 75 | } |
| 76 | |
| 77 | /** |
| 78 | * Steps a test can fail at, to leave a run half-done: migrating, after the |
| 79 | * S3 lease is fenced and after the copy is checked but before it commits; |
| 80 | * rolling back, after the stage is switched back but before its DynamoDB |
| 81 | * items are deleted. |
| 82 | */ |
| 83 | export type MigrationStep = "fenced" | "copied" | "switched"; |
| 84 | |
| 85 | export interface MigrateOptions extends Omit<StateMigrationRequest, "rollback"> { |
| 86 | /** Runs after each step; a failure stops the stage there. */ |
| 87 | readonly afterStep?: ( |
| 88 | step: MigrationStep, |
| 89 | stage: { stack: string; stage: string }, |
| 90 | ) => Effect.Effect<void, StateStoreError>; |
| 91 | } |
| 92 | |
| 93 | const describeLocation = (location: LegacyLocation) => `s3://${location.bucket}/${location.prefix}`; |
| 94 | |
| 95 | /** Name an S3 failure in the store's error type. */ |
| 96 | const toStoreError = (cause: { readonly message?: string; readonly _tag?: string }) => |
| 97 | new StateStoreError({ message: cause.message ?? `S3 state store error: ${cause._tag}` }); |
| 98 | |
| 99 | /** |
| 100 | * A stage's result whatever happens to it: a lease another deploy holds is |
| 101 | * a blocked stage, and any other failure is reported for that stage rather |
| 102 | * than stopping the others. |
| 103 | */ |
| 104 | const settle = |
| 105 | (stack: string, stage: string) => |
| 106 | <R>(self: Effect.Effect<StageMigration, StateStoreError, R>) => |
| 107 | self.pipe( |
| 108 | Effect.catchTag("StateStoreError", (error) => |
| 109 | Effect.succeed<StageMigration>({ |
| 110 | stack, |
| 111 | stage, |
| 112 | status: isLeaseHeld(error) ? "blocked" : "failed", |
| 113 | reason: error.message, |
| 114 | }), |
| 115 | ), |
| 116 | ); |
| 117 | |
| 118 | const heldInS3 = (stack: string, stage: string, expiresAt: number): StageMigration => ({ |
| 119 | stack, |
| 120 | stage, |
| 121 | status: "blocked", |
| 122 | reason: `a deploy holds the S3 state lock until ${new Date(expiresAt).toISOString()}`, |
| 123 | }); |
| 124 | |
| 125 | const migrateStage = Effect.fn(function* ( |
| 126 | store: DynamoDBStateService, |
| 127 | legacy: LegacyLocation, |
| 128 | table: string, |
| 129 | stack: string, |
| 130 | stage: string, |
| 131 | options: MigrateOptions, |
| 132 | ) { |
| 133 | const { migration } = store; |
| 134 | const s3 = <A, E extends { readonly message?: string; readonly _tag?: string }, R>( |
| 135 | effect: Effect.Effect<A, E, R>, |
| 136 | ) => effect.pipe(Effect.mapError(toStoreError)); |
| 137 | // An entry only appears or goes under the lease, so a stage DynamoDB owns |
| 138 | // needs none. Its bucket policy is written again, in case a crash after |
| 139 | // the commit left it out. |
| 140 | const alreadyMigrated = Effect.gen(function* () { |
| 141 | yield* s3(setStageAccess(legacy, stack, stage, "migrated")); |
| 142 | return { stack, stage, status: "already-migrated" } satisfies StageMigration; |
| 143 | }); |
| 144 | if (ownsStage(yield* migration.status(stack, stage))) return yield* alreadyMigrated; |
| 145 | return yield* migration.exclusive(stack, stage, (lease: HeldLease) => |
| 146 | Effect.gen(function* () { |
| 147 | if (ownsStage(yield* migration.status(stack, stage))) return yield* alreadyMigrated; |
| 148 | const fence = yield* s3(fenceLegacyStage(legacy, stack, stage, table)); |
| 149 | if (!fence.fenced) return heldInS3(stack, stage, fence.expiresAt); |
| 150 | yield* s3(setStageAccess(legacy, stack, stage, "fenced")); |
| 151 | yield* options.afterStep?.("fenced", { stack, stage }) ?? Effect.void; |
| 152 | const contents = yield* s3(readLegacyStage(legacy, stack, stage)); |
| 153 | yield* migration.commitStage( |
| 154 | stack, |
| 155 | stage, |
| 156 | lease, |
| 157 | contents, |
| 158 | describeLocation(legacy), |
| 159 | options.afterStep?.("copied", { stack, stage }), |
| 160 | ); |
| 161 | yield* s3(setStageAccess(legacy, stack, stage, "migrated")); |
| 162 | return { |
| 163 | stack, |
| 164 | stage, |
| 165 | status: "migrated", |
| 166 | records: contents.records.size, |
| 167 | } satisfies StageMigration; |
| 168 | }), |
| 169 | ); |
| 170 | }); |
| 171 | |
| 172 | const rollbackStage = Effect.fn(function* ( |
| 173 | store: DynamoDBStateService, |
| 174 | legacy: LegacyLocation, |
| 175 | table: string, |
| 176 | stack: string, |
| 177 | stage: string, |
| 178 | options: MigrateOptions, |
| 179 | ) { |
| 180 | const { migration } = store; |
| 181 | const s3 = <A, E extends { readonly message?: string; readonly _tag?: string }, R>( |
| 182 | effect: Effect.Effect<A, E, R>, |
| 183 | ) => effect.pipe(Effect.mapError(toStoreError)); |
| 184 | return yield* migration.exclusive(stack, stage, (lease: HeldLease) => |
| 185 | Effect.gen(function* () { |
| 186 | const status = yield* migration.status(stack, stage); |
| 187 | if (status === undefined) { |
| 188 | return { stack, stage, status: "not-migrated" } satisfies StageMigration; |
| 189 | } |
| 190 | // Reading and writing the S3 copy needs the bucket policy lifted. |
| 191 | yield* s3(setStageAccess(legacy, stack, stage, "open")); |
| 192 | let records: number | undefined; |
| 193 | // A `rolled-back` stage was switched back by a run that stopped |
| 194 | // before cleaning up: only the cleanup is left. |
| 195 | if (status !== "rolled-back") { |
| 196 | // A stage that started in DynamoDB has no fence yet: take one, so |
| 197 | // no S3 store writes the stage while it is copied back. |
| 198 | const fence = yield* s3(fenceLegacyStage(legacy, stack, stage, table)); |
| 199 | if (!fence.fenced) return heldInS3(stack, stage, fence.expiresAt); |
| 200 | // A destroyed stage has no contents, which empties its S3 copy. |
| 201 | const contents = yield* migration.readStage(stack, stage); |
| 202 | yield* s3(writeLegacyStage(legacy, stack, stage, contents)); |
| 203 | // The switch back. Past it, the S3 copy is the stage's state. |
| 204 | yield* migration.markRolledBack(stack, stage, lease); |
| 205 | records = contents.records.size; |
| 206 | yield* options.afterStep?.("switched", { stack, stage }) ?? Effect.void; |
| 207 | } |
| 208 | yield* migration.deleteStageItems(stack, stage, lease); |
| 209 | yield* s3(unfenceLegacyStage(legacy, stack, stage)); |
| 210 | return { |
| 211 | stack, |
| 212 | stage, |
| 213 | status: "rolled-back", |
| 214 | ...(records === undefined ? {} : { records }), |
| 215 | } satisfies StageMigration; |
| 216 | }), |
| 217 | ); |
| 218 | }); |
| 219 | |
| 220 | /** Keep the stages a request selects: all of them, one stack's, or one. */ |
| 221 | const selected = |
| 222 | (options: { stack?: string; stage?: string }) => (entry: { stack: string; stage: string }) => |
| 223 | (options.stack === undefined || entry.stack === options.stack) && |
| 224 | (options.stage === undefined || entry.stage === options.stage); |
| 225 | |
| 226 | const legacyStages = (legacy: LegacyLocation) => |
| 227 | listLegacyStacks(legacy).pipe( |
| 228 | Effect.flatMap((stacks) => |
| 229 | Effect.forEach( |
| 230 | stacks, |
| 231 | (stack) => |
| 232 | listLegacyStages(legacy, stack).pipe( |
| 233 | Effect.map((stages) => stages.map((stage) => ({ stack, stage }))), |
| 234 | ), |
| 235 | { concurrency: 8 }, |
| 236 | ), |
| 237 | ), |
| 238 | Effect.map((stages) => stages.flat()), |
| 239 | Effect.mapError(toStoreError), |
| 240 | ); |
| 241 | |
| 242 | const requireLegacy = <T extends { legacy: LegacyLocation | undefined }>(target: T) => |
| 243 | target.legacy === undefined |
| 244 | ? Effect.fail( |
| 245 | new StateStoreError({ message: "This DynamoDB state store has no S3 store to migrate" }), |
| 246 | ) |
| 247 | : Effect.succeed({ ...target, legacy: target.legacy }); |
| 248 | |
| 249 | /** |
| 250 | * Move every stage of the S3 store (or the selected ones) to DynamoDB, |
| 251 | * creating the table if it is missing. Each stage reports its own result; |
| 252 | * run it again to finish the ones blocked or failed. |
| 253 | */ |
| 254 | export const migrateToDynamoDB = Effect.fn("AWS.state.migrate")(function* ( |
| 255 | store: DynamoDBStateService, |
| 256 | options: MigrateOptions = {}, |
| 257 | ) { |
| 258 | const { legacy, table } = yield* store.migration.provision.pipe(Effect.flatMap(requireLegacy)); |
| 259 | const stages = (yield* legacyStages(legacy)).filter(selected(options)); |
| 260 | return yield* Effect.forEach( |
| 261 | stages, |
| 262 | ({ stack, stage }) => |
| 263 | migrateStage(store, legacy, table, stack, stage, options).pipe(settle(stack, stage)), |
| 264 | { concurrency: 4 }, |
| 265 | ); |
| 266 | }); |
| 267 | |
| 268 | /** |
| 269 | * Move every DynamoDB stage (or the selected ones) back to the S3 store, |
| 270 | * for going back to a version of alchemy that only has the S3 store. |
| 271 | */ |
| 272 | export const rollbackToS3 = Effect.fn("AWS.state.rollback")(function* ( |
| 273 | store: DynamoDBStateService, |
| 274 | options: MigrateOptions = {}, |
| 275 | ) { |
| 276 | const { legacy, table } = yield* store.migration.names.pipe(Effect.flatMap(requireLegacy)); |
| 277 | const stages = (yield* store.migration.catalog()).filter(selected(options)); |
| 278 | return yield* Effect.forEach( |
| 279 | stages, |
| 280 | ({ stack, stage }) => |
| 281 | rollbackStage(store, legacy, table, stack, stage, options).pipe(settle(stack, stage)), |
| 282 | { concurrency: 4 }, |
| 283 | ); |
| 284 | }); |