The best way to control Claude Code, Codex and more from any computer or phone

main
ts 284 lines 10.8 KB
Raw
1 import type { StateMigration, StateMigrationRequest } from "alchemy/State/State";
2 import { StateStoreError } from "alchemy/State/State";
3 import * as Effect from "effect/Effect";
4 import type { DynamoDBStateService, HeldLease } from "./DynamoDBState.ts";
5 import {
6 fenceLegacyStage,
7 isLeaseHeld,
8 type LegacyLocation,
9 listLegacyStacks,
10 listLegacyStages,
11 readLegacyStage,
12 setStageAccess,
13 unfenceLegacyStage,
14 writeLegacyStage,
15 } from "./LegacyLayout.ts";
16
17 /**
18 * Moving stages from the S3 state store to the DynamoDB one, and back.
19 *
20 * Each stage moves on its own, under both stores' leases:
21 *
22 * 1. Hold the stage's DynamoDB lease. A stage DynamoDB owns has moved.
23 * 2. Fence the stage in S3: replace its lease with one that never expires,
24 * which every S3 store since 2.0.0-beta.84 reads as held, and deny
25 * writes to its objects in the bucket policy, which stops the versions
26 * before it. A deploy that holds the lease keeps it, and the stage is
27 * reported as blocked.
28 * 3. Copy every record and the output, and delete what an earlier,
29 * interrupted copy left that the source no longer has.
30 * 4. Read the copy back and compare it with the source.
31 * 5. Enter the stage in the DynamoDB catalog, on condition the lease is
32 * still ours. This write is the switch: the DynamoDB store refuses a
33 * stage the S3 store has until it is there.
34 * 6. Deny reads of the stage's objects too, so any older version fails at
35 * its first read instead of deploying from the copy left behind. S3 can
36 * take minutes to enforce this second policy update everywhere; writes
37 * stay denied throughout, so an older version that reads the copy in
38 * that window still can't record anything.
39 *
40 * A failure at any step leaves the S3 copy authoritative and, once fenced,
41 * unchanged, so running the migration again converges. The S3 objects
42 * are kept.
43 */
44
45 /**
46 * A stage's entry in the DynamoDB catalog:
47 *
48 * - `active`: DynamoDB holds the stage.
49 * - `destroyed`: the stage was destroyed in DynamoDB. The entry stays, so an
50 * S3 copy from before the migration doesn't come back.
51 * - `rolled-back`: moved back; the S3 store holds the stage again.
52 */
53 export type CatalogStatus = "active" | "destroyed" | "rolled-back";
54
55 /** Whether DynamoDB is the stage's store, whatever the S3 store still has. */
56 export const ownsStage = (status: CatalogStatus | undefined) =>
57 status === "active" || status === "destroyed";
58
59 export type StageMigrationStatus =
60 /** Moved by this run. */
61 | "migrated"
62 /** Moved by an earlier run, or never in the S3 store. */
63 | "already-migrated"
64 /** Moved back to the S3 store by this run. */
65 | "rolled-back"
66 /** Not in DynamoDB, so there is nothing to roll back. */
67 | "not-migrated"
68 /** Another deploy holds the stage. */
69 | "blocked"
70 /** Moving the stage failed; the reason says why. */
71 | "failed";
72
73 export interface StageMigration extends StateMigration {
74 readonly status: StageMigrationStatus;
75 }
76
77 /**
78 * Steps a test can fail at, to leave a run half-done: migrating, after the
79 * S3 lease is fenced and after the copy is checked but before it commits;
80 * rolling back, after the stage is switched back but before its DynamoDB
81 * items are deleted.
82 */
83 export type MigrationStep = "fenced" | "copied" | "switched";
84
85 export interface MigrateOptions extends Omit<StateMigrationRequest, "rollback"> {
86 /** Runs after each step; a failure stops the stage there. */
87 readonly afterStep?: (
88 step: MigrationStep,
89 stage: { stack: string; stage: string },
90 ) => Effect.Effect<void, StateStoreError>;
91 }
92
93 const describeLocation = (location: LegacyLocation) => `s3://${location.bucket}/${location.prefix}`;
94
95 /** Name an S3 failure in the store's error type. */
96 const toStoreError = (cause: { readonly message?: string; readonly _tag?: string }) =>
97 new StateStoreError({ message: cause.message ?? `S3 state store error: ${cause._tag}` });
98
99 /**
100 * A stage's result whatever happens to it: a lease another deploy holds is
101 * a blocked stage, and any other failure is reported for that stage rather
102 * than stopping the others.
103 */
104 const settle =
105 (stack: string, stage: string) =>
106 <R>(self: Effect.Effect<StageMigration, StateStoreError, R>) =>
107 self.pipe(
108 Effect.catchTag("StateStoreError", (error) =>
109 Effect.succeed<StageMigration>({
110 stack,
111 stage,
112 status: isLeaseHeld(error) ? "blocked" : "failed",
113 reason: error.message,
114 }),
115 ),
116 );
117
118 const heldInS3 = (stack: string, stage: string, expiresAt: number): StageMigration => ({
119 stack,
120 stage,
121 status: "blocked",
122 reason: `a deploy holds the S3 state lock until ${new Date(expiresAt).toISOString()}`,
123 });
124
125 const migrateStage = Effect.fn(function* (
126 store: DynamoDBStateService,
127 legacy: LegacyLocation,
128 table: string,
129 stack: string,
130 stage: string,
131 options: MigrateOptions,
132 ) {
133 const { migration } = store;
134 const s3 = <A, E extends { readonly message?: string; readonly _tag?: string }, R>(
135 effect: Effect.Effect<A, E, R>,
136 ) => effect.pipe(Effect.mapError(toStoreError));
137 // An entry only appears or goes under the lease, so a stage DynamoDB owns
138 // needs none. Its bucket policy is written again, in case a crash after
139 // the commit left it out.
140 const alreadyMigrated = Effect.gen(function* () {
141 yield* s3(setStageAccess(legacy, stack, stage, "migrated"));
142 return { stack, stage, status: "already-migrated" } satisfies StageMigration;
143 });
144 if (ownsStage(yield* migration.status(stack, stage))) return yield* alreadyMigrated;
145 return yield* migration.exclusive(stack, stage, (lease: HeldLease) =>
146 Effect.gen(function* () {
147 if (ownsStage(yield* migration.status(stack, stage))) return yield* alreadyMigrated;
148 const fence = yield* s3(fenceLegacyStage(legacy, stack, stage, table));
149 if (!fence.fenced) return heldInS3(stack, stage, fence.expiresAt);
150 yield* s3(setStageAccess(legacy, stack, stage, "fenced"));
151 yield* options.afterStep?.("fenced", { stack, stage }) ?? Effect.void;
152 const contents = yield* s3(readLegacyStage(legacy, stack, stage));
153 yield* migration.commitStage(
154 stack,
155 stage,
156 lease,
157 contents,
158 describeLocation(legacy),
159 options.afterStep?.("copied", { stack, stage }),
160 );
161 yield* s3(setStageAccess(legacy, stack, stage, "migrated"));
162 return {
163 stack,
164 stage,
165 status: "migrated",
166 records: contents.records.size,
167 } satisfies StageMigration;
168 }),
169 );
170 });
171
172 const rollbackStage = Effect.fn(function* (
173 store: DynamoDBStateService,
174 legacy: LegacyLocation,
175 table: string,
176 stack: string,
177 stage: string,
178 options: MigrateOptions,
179 ) {
180 const { migration } = store;
181 const s3 = <A, E extends { readonly message?: string; readonly _tag?: string }, R>(
182 effect: Effect.Effect<A, E, R>,
183 ) => effect.pipe(Effect.mapError(toStoreError));
184 return yield* migration.exclusive(stack, stage, (lease: HeldLease) =>
185 Effect.gen(function* () {
186 const status = yield* migration.status(stack, stage);
187 if (status === undefined) {
188 return { stack, stage, status: "not-migrated" } satisfies StageMigration;
189 }
190 // Reading and writing the S3 copy needs the bucket policy lifted.
191 yield* s3(setStageAccess(legacy, stack, stage, "open"));
192 let records: number | undefined;
193 // A `rolled-back` stage was switched back by a run that stopped
194 // before cleaning up: only the cleanup is left.
195 if (status !== "rolled-back") {
196 // A stage that started in DynamoDB has no fence yet: take one, so
197 // no S3 store writes the stage while it is copied back.
198 const fence = yield* s3(fenceLegacyStage(legacy, stack, stage, table));
199 if (!fence.fenced) return heldInS3(stack, stage, fence.expiresAt);
200 // A destroyed stage has no contents, which empties its S3 copy.
201 const contents = yield* migration.readStage(stack, stage);
202 yield* s3(writeLegacyStage(legacy, stack, stage, contents));
203 // The switch back. Past it, the S3 copy is the stage's state.
204 yield* migration.markRolledBack(stack, stage, lease);
205 records = contents.records.size;
206 yield* options.afterStep?.("switched", { stack, stage }) ?? Effect.void;
207 }
208 yield* migration.deleteStageItems(stack, stage, lease);
209 yield* s3(unfenceLegacyStage(legacy, stack, stage));
210 return {
211 stack,
212 stage,
213 status: "rolled-back",
214 ...(records === undefined ? {} : { records }),
215 } satisfies StageMigration;
216 }),
217 );
218 });
219
220 /** Keep the stages a request selects: all of them, one stack's, or one. */
221 const selected =
222 (options: { stack?: string; stage?: string }) => (entry: { stack: string; stage: string }) =>
223 (options.stack === undefined || entry.stack === options.stack) &&
224 (options.stage === undefined || entry.stage === options.stage);
225
226 const legacyStages = (legacy: LegacyLocation) =>
227 listLegacyStacks(legacy).pipe(
228 Effect.flatMap((stacks) =>
229 Effect.forEach(
230 stacks,
231 (stack) =>
232 listLegacyStages(legacy, stack).pipe(
233 Effect.map((stages) => stages.map((stage) => ({ stack, stage }))),
234 ),
235 { concurrency: 8 },
236 ),
237 ),
238 Effect.map((stages) => stages.flat()),
239 Effect.mapError(toStoreError),
240 );
241
242 const requireLegacy = <T extends { legacy: LegacyLocation | undefined }>(target: T) =>
243 target.legacy === undefined
244 ? Effect.fail(
245 new StateStoreError({ message: "This DynamoDB state store has no S3 store to migrate" }),
246 )
247 : Effect.succeed({ ...target, legacy: target.legacy });
248
249 /**
250 * Move every stage of the S3 store (or the selected ones) to DynamoDB,
251 * creating the table if it is missing. Each stage reports its own result;
252 * run it again to finish the ones blocked or failed.
253 */
254 export const migrateToDynamoDB = Effect.fn("AWS.state.migrate")(function* (
255 store: DynamoDBStateService,
256 options: MigrateOptions = {},
257 ) {
258 const { legacy, table } = yield* store.migration.provision.pipe(Effect.flatMap(requireLegacy));
259 const stages = (yield* legacyStages(legacy)).filter(selected(options));
260 return yield* Effect.forEach(
261 stages,
262 ({ stack, stage }) =>
263 migrateStage(store, legacy, table, stack, stage, options).pipe(settle(stack, stage)),
264 { concurrency: 4 },
265 );
266 });
267
268 /**
269 * Move every DynamoDB stage (or the selected ones) back to the S3 store,
270 * for going back to a version of alchemy that only has the S3 store.
271 */
272 export const rollbackToS3 = Effect.fn("AWS.state.rollback")(function* (
273 store: DynamoDBStateService,
274 options: MigrateOptions = {},
275 ) {
276 const { legacy, table } = yield* store.migration.names.pipe(Effect.flatMap(requireLegacy));
277 const stages = (yield* store.migration.catalog()).filter(selected(options));
278 return yield* Effect.forEach(
279 stages,
280 ({ stack, stage }) =>
281 rollbackStage(store, legacy, table, stack, stage, options).pipe(settle(stack, stage)),
282 { concurrency: 4 },
283 );
284 });