The best way to control Claude Code, Codex and more from any computer or phone

main
ts 493 lines 18.7 KB
Raw
1 import * as Clock from "effect/Clock";
2 import * as Duration from "effect/Duration";
3 import * as Effect from "effect/Effect";
4 import * as Schedule from "effect/Schedule";
5 import * as Scope from "effect/Scope";
6 import * as Semaphore from "effect/Semaphore";
7 import { decodeFqn, encodeFqn } from "../FQN.ts";
8 import { STATE_STORE_VERSION } from "./HttpStateApi.ts";
9 import { StateStoreError, type PersistedState, type StateService } from "./State.ts";
10 import { encodeState, reviveState } from "./StateEncoding.ts";
11
12 /**
13 * A state store over an object storage bucket (S3, Cloud Storage, …).
14 *
15 * The store is laid out exactly like the local state store's file tree,
16 * with the bucket (plus an optional prefix) in place of `.alchemy/state`:
17 *
18 * ```
19 * {prefix}{stack}/{stage}/{fqn}.json
20 * {prefix}{stack}/{stage}/__stack_output__.json
21 * {prefix}{stack}/{stage}/__lease__.json
22 * ```
23 *
24 * Concurrent deploys of one stack and stage share the `__lease__.json`
25 * object, written with conditional puts. The second deploy fails
26 * immediately instead of mixing rows. The holder refreshes the lease until
27 * the store closes; a killed process stops refreshing, the lease expires,
28 * and the next deploy can take it.
29 *
30 * Each backend supplies a {@link BucketObjects}: plain object operations
31 * plus a version token (an S3 ETag, a Cloud Storage generation) for the
32 * conditional writes the lease needs.
33 */
34
35 /** The bookkeeping object that stores a stack's resolved output. */
36 const OUTPUT_FILE = "__stack_output__.json";
37
38 /**
39 * Per `(stack, stage)` lease. A delete of the stage leaves it in place so
40 * the lock outlives the objects it was guarding; the store deletes it on
41 * close.
42 */
43 /** The lease token `alchemy state migrate` leaves on a stage it moved to DynamoDB. */
44 const MIGRATED_TOKEN = "migrated:dynamodb";
45
46 const LEASE_FILE = "__lease__.json";
47
48 /** How long a lease stays valid without a refresh. */
49 const LEASE_TTL_MS = 60_000;
50
51 /** How long a passing lease check is trusted. A failure is never cached. */
52 const LEASE_CHECK_TTL_MS = 5_000;
53
54 /** A write condition, in terms of the backend's version token. */
55 export type WriteCondition =
56 /** Only create the object; fail if it exists. */
57 | { readonly ifAbsent: true }
58 /** Only overwrite the object at exactly this version. */
59 | { readonly ifVersion: string };
60
61 /**
62 * The object operations a bucket backend implements. Every operation is
63 * self-contained (`R = never`) and fails with {@link StateStoreError}.
64 */
65 export interface BucketObjects {
66 /** Every key under `prefix`, across pagination. */
67 readonly listKeys: (prefix: string) => Effect.Effect<ReadonlyArray<string>, StateStoreError>;
68 /**
69 * The immediate "directory" names under `prefix` (delimiter `/`), without
70 * the prefix or the trailing slash.
71 */
72 readonly listChildren: (prefix: string) => Effect.Effect<ReadonlyArray<string>, StateStoreError>;
73 /** An object's text and version, or `undefined` when it doesn't exist. */
74 readonly read: (
75 key: string,
76 ) => Effect.Effect<
77 { readonly text: string; readonly version: string } | undefined,
78 StateStoreError
79 >;
80 /**
81 * Write an object. Returns its new version, or `undefined` when
82 * `condition` did not hold.
83 */
84 readonly write: (
85 key: string,
86 body: string,
87 condition?: WriteCondition,
88 ) => Effect.Effect<string | undefined, StateStoreError>;
89 /**
90 * Delete an object, only at `ifVersion` when given. Deleting a missing
91 * object, or one at another version, succeeds without deleting.
92 */
93 readonly delete: (key: string, ifVersion?: string) => Effect.Effect<void, StateStoreError>;
94 /** Delete many objects. Missing objects are ignored. */
95 readonly deleteMany: (keys: ReadonlyArray<string>) => Effect.Effect<void, StateStoreError>;
96 }
97
98 export interface BucketStateOptions {
99 /** The store's id, e.g. `"s3"` or `"gcs"`. */
100 readonly id: string;
101 /** How errors name the store, e.g. `"S3"` or `"Cloud Storage"`. */
102 readonly label: string;
103 /** Key prefix within the bucket. A trailing `/` is added. */
104 readonly prefix?: string;
105 /**
106 * The bucket's objects. Runs once, on the first state operation, and is
107 * cached: ensure the bucket exists here, never at construction.
108 */
109 readonly objects: Effect.Effect<BucketObjects, StateStoreError>;
110 }
111
112 /**
113 * Tokens of the leases this process holds, so a conflict with one of them
114 * can say the holder is an earlier, still-open build in the same process.
115 */
116 const heldInProcess = new Set<string>();
117
118 interface LeaseRecord {
119 readonly token: string;
120 readonly expiresAt: number;
121 }
122
123 interface HeldLease {
124 readonly checkLive: Effect.Effect<void, StateStoreError>;
125 }
126
127 /**
128 * A passing check is trusted for `ttlMs`, so a burst of operations does
129 * one round-trip. A failing check is never cached.
130 */
131 const amortizeCheck = (
132 checkLive: Effect.Effect<void, StateStoreError>,
133 ttlMs: number,
134 ): Effect.Effect<void, StateStoreError> => {
135 let lastOkAt: number | undefined;
136 return Effect.suspend(() => {
137 if (lastOkAt !== undefined && Date.now() - lastOkAt < ttlMs) {
138 return Effect.void;
139 }
140 return checkLive.pipe(
141 Effect.tap(() =>
142 Effect.sync(() => {
143 lastOkAt = Date.now();
144 }),
145 ),
146 );
147 });
148 };
149
150 const parseLease = (text: string): LeaseRecord => {
151 let parsed: unknown;
152 try {
153 parsed = JSON.parse(text);
154 } catch {
155 parsed = undefined;
156 }
157 return typeof parsed === "object" &&
158 parsed !== null &&
159 "token" in parsed &&
160 "expiresAt" in parsed &&
161 typeof parsed.token === "string" &&
162 typeof parsed.expiresAt === "number"
163 ? { token: parsed.token, expiresAt: parsed.expiresAt }
164 : { token: "", expiresAt: 0 };
165 };
166
167 /**
168 * Construct a bucket-backed {@link StateService}. Leases are bound to the
169 * ambient scope; callers that need them to outlive the first operation
170 * bind this effect to a longer scope with `Scope.provide`.
171 */
172 export const makeBucketState = (options: BucketStateOptions) =>
173 Effect.gen(function* () {
174 const leaseScope = yield* Effect.scope;
175 const prefix = options.prefix ? `${options.prefix.replace(/\/+$/, "")}/` : "";
176 const objects = yield* Effect.cached(options.objects);
177 const use = <A>(
178 f: (objects: BucketObjects) => Effect.Effect<A, StateStoreError>,
179 ): Effect.Effect<A, StateStoreError> => objects.pipe(Effect.flatMap(f));
180
181 const stagePrefix = ({ stack, stage }: { stack: string; stage: string }) =>
182 `${prefix}${stack}/${stage}/`;
183 const resourceKey = (request: { stack: string; stage: string; fqn: string }) =>
184 `${stagePrefix(request)}${encodeFqn(request.fqn)}.json`;
185 const outputKey = (request: { stack: string; stage: string }) =>
186 `${stagePrefix(request)}${OUTPUT_FILE}`;
187 const leaseKey = (request: { stack: string; stage: string }) =>
188 `${stagePrefix(request)}${LEASE_FILE}`;
189
190 const readJson = <T>(key: string) =>
191 use((o) => o.read(key)).pipe(
192 Effect.flatMap((object) =>
193 object === undefined
194 ? Effect.undefined
195 : Effect.try({
196 try: () => JSON.parse(object.text, reviveState) as T,
197 catch: (cause) =>
198 new StateStoreError({
199 message: `Failed to parse state object '${key}'`,
200 cause: cause instanceof Error ? cause : undefined,
201 }),
202 }),
203 ),
204 );
205
206 const writeJson = (key: string, value: unknown) =>
207 use((o) => o.write(key, JSON.stringify(encodeState(value), null, 2)));
208
209 /** Delete every object under `keyPrefix`, except leases. Idempotent. */
210 const deleteAll = (keyPrefix: string) =>
211 use((o) =>
212 o.listKeys(keyPrefix).pipe(
213 // Leave the lease object. Releasing it here would open the stage
214 // before this delete finishes, and the store still holds the lock.
215 Effect.map((keys) => keys.filter((key) => !key.endsWith(`/${LEASE_FILE}`))),
216 Effect.flatMap((keys) => (keys.length === 0 ? Effect.void : o.deleteMany(keys))),
217 ),
218 );
219
220 const heldError = (stack: string, stage: string, holder: string | undefined) =>
221 // The message always starts "another deploy holds": callers match on it
222 // (the AWS migration tells a held stage from a failed one by it).
223 new StateStoreError({
224 message:
225 holder !== undefined && heldInProcess.has(holder)
226 ? `another deploy holds the ${options.label} state lock '${stack}/${stage}': this process holds it, from an earlier operation on this stage that is still open. Run each operation in its own Effect.scoped so its build closes first.`
227 : `another deploy holds the ${options.label} state lock '${stack}/${stage}'`,
228 });
229
230 const lostError = (stack: string, stage: string) =>
231 new StateStoreError({
232 message: `the ${options.label} state lock '${stack}/${stage}' was lost mid-run; refusing to continue unlocked`,
233 });
234
235 const readLease = (key: string) =>
236 use((o) => o.read(key)).pipe(
237 Effect.map((object) =>
238 object === undefined
239 ? undefined
240 : { version: object.version, record: parseLease(object.text) },
241 ),
242 );
243
244 const putLease = (key: string, record: LeaseRecord, condition: WriteCondition) =>
245 use((o) => o.write(key, JSON.stringify(record), condition));
246
247 const acquireLease = (stack: string, stage: string) =>
248 Effect.gen(function* () {
249 const key = leaseKey({ stack, stage });
250 const token = yield* Effect.sync(() => crypto.randomUUID());
251 const now = yield* Clock.currentTimeMillis;
252 const fresh: LeaseRecord = { token, expiresAt: now + LEASE_TTL_MS };
253 let version = yield* putLease(key, fresh, { ifAbsent: true });
254 if (version === undefined) {
255 const existing = yield* readLease(key);
256 const stillHeld = existing === undefined || existing.record.expiresAt > now;
257 if (existing?.record.token === MIGRATED_TOKEN) {
258 return yield* new StateStoreError({
259 message: `${heldError(stack, stage, undefined).message}: the stage moved to a DynamoDB state store. Deploy it with a version of alchemy that has AWS.state() on DynamoDB, or run 'alchemy state migrate --rollback' with that version to move it back.`,
260 });
261 }
262 if (stillHeld) {
263 return yield* heldError(stack, stage, existing?.record.token);
264 }
265 const replaced = yield* putLease(key, fresh, { ifVersion: existing.version });
266 if (replaced === undefined) {
267 return yield* heldError(stack, stage, undefined);
268 }
269 version = replaced;
270 }
271
272 heldInProcess.add(token);
273 const cursor = { version, lost: false };
274 const gate = Semaphore.makeUnsafe(1);
275
276 const renew = Semaphore.withPermits(
277 gate,
278 1,
279 )(
280 Effect.gen(function* () {
281 if (cursor.lost) return;
282 const renewedAt = yield* Clock.currentTimeMillis;
283 const next = yield* putLease(
284 key,
285 { token, expiresAt: renewedAt + LEASE_TTL_MS },
286 { ifVersion: cursor.version },
287 );
288 if (next === undefined) cursor.lost = true;
289 else cursor.version = next;
290 }),
291 ).pipe(
292 Effect.retry({
293 while: () => !cursor.lost,
294 schedule: Schedule.spaced("1 second"),
295 times: 3,
296 }),
297 Effect.catch(() =>
298 Effect.sync(() => {
299 cursor.lost = true;
300 }),
301 ),
302 );
303
304 const verify = Semaphore.withPermits(
305 gate,
306 1,
307 )(
308 Effect.gen(function* () {
309 if (cursor.lost) return yield* lostError(stack, stage);
310 const existing = yield* readLease(key);
311 if (existing === undefined || existing.record.token !== token) {
312 cursor.lost = true;
313 return yield* lostError(stack, stage);
314 }
315 const observedAt = yield* Clock.currentTimeMillis;
316 if (existing.record.expiresAt <= observedAt) {
317 // The lease lapsed without anyone taking it: a refresh queued
318 // behind a large deploy's writes missed its window. Take it
319 // back at the version we read, so a deploy that took it in
320 // the meantime wins and this one stops.
321 const reclaimed = yield* putLease(
322 key,
323 { token, expiresAt: observedAt + LEASE_TTL_MS },
324 { ifVersion: existing.version },
325 );
326 if (reclaimed === undefined) {
327 cursor.lost = true;
328 return yield* lostError(stack, stage);
329 }
330 cursor.version = reclaimed;
331 return;
332 }
333 cursor.version = existing.version;
334 }),
335 );
336
337 // Interrupt the refresh before releasing, so a refresh cannot
338 // land after the lease has been released.
339 yield* Scope.addFinalizer(
340 leaseScope,
341 Semaphore.withPermits(
342 gate,
343 1,
344 )(
345 Effect.gen(function* () {
346 cursor.lost = true;
347 heldInProcess.delete(token);
348 // Read the version we actually hold. A refresh that landed
349 // after the cursor was last stored would make a release
350 // against the old version a no-op, and the lease would stay.
351 const existing = yield* readLease(key);
352 if (existing === undefined || existing.record.token !== token) return;
353 // Delete rather than expire: a leftover lease object would keep
354 // a destroyed stage listed by `listStages`.
355 yield* use((o) => o.delete(key, existing.version));
356 }).pipe(Effect.ignore),
357 ),
358 );
359 yield* renew.pipe(
360 Effect.delay(Duration.millis(Math.floor(LEASE_TTL_MS / 3))),
361 Effect.forever,
362 Effect.interruptible,
363 Effect.forkIn(leaseScope),
364 );
365
366 return { checkLive: amortizeCheck(verify, LEASE_CHECK_TTL_MS) } satisfies HeldLease;
367 });
368
369 // One lease per (stack, stage), taken on the first operation and held
370 // for the store's lifetime. The mutex keeps two concurrent first
371 // operations in this process from contending with each other.
372 const leaseMutex = Semaphore.makeUnsafe(1);
373 const leases = new Map<string, Effect.Effect<HeldLease, StateStoreError>>();
374
375 const leaseFor = (stack: string, stage: string): Effect.Effect<HeldLease, StateStoreError> =>
376 Semaphore.withPermits(
377 leaseMutex,
378 1,
379 )(
380 Effect.gen(function* () {
381 const key = `${stack}/${stage}`;
382 const existing = leases.get(key);
383 if (existing !== undefined) return existing;
384 // Drop a failed acquire. Caching it would make the next operation
385 // report the same holder after that holder has gone.
386 const cached = yield* Effect.cached(
387 acquireLease(stack, stage).pipe(
388 Effect.tapError(() =>
389 Effect.sync(() => {
390 if (leases.get(key) === cached) {
391 leases.delete(key);
392 }
393 }),
394 ),
395 ),
396 );
397 leases.set(key, cached);
398 return cached;
399 }),
400 ).pipe(Effect.flatMap((lease) => lease));
401
402 const guarded = <A>(
403 request: { stack: string; stage: string },
404 op: Effect.Effect<A, StateStoreError>,
405 ): Effect.Effect<A, StateStoreError> =>
406 leaseFor(request.stack, request.stage).pipe(
407 Effect.flatMap((lease) => lease.checkLive),
408 Effect.andThen(op),
409 );
410
411 const verifyHeldLeases: Effect.Effect<void, StateStoreError> = Effect.suspend(() =>
412 Effect.forEach(
413 Array.from(leases.values()),
414 (lease) => lease.pipe(Effect.flatMap((held) => held.checkLive)),
415 { discard: true },
416 ),
417 );
418
419 const listChildren = (keyPrefix: string) =>
420 use((o) => o.listChildren(keyPrefix)).pipe(Effect.map((names) => Array.from(names)));
421
422 const state: StateService = {
423 id: options.id,
424 getVersion: () => Effect.succeed(STATE_STORE_VERSION),
425 listStacks: () => verifyHeldLeases.pipe(Effect.andThen(listChildren(prefix))),
426 listStages: (stack: string) =>
427 verifyHeldLeases.pipe(Effect.andThen(listChildren(`${prefix}${stack}/`))),
428 get: (request) => guarded(request, readJson<PersistedState>(resourceKey(request))),
429 getReplacedResources: Effect.fn(function* (request) {
430 return (yield* Effect.forEach(yield* state.list(request), (fqn) =>
431 state.get({
432 stack: request.stack,
433 stage: request.stage,
434 fqn,
435 }),
436 )).filter((r) => r?.status === "replaced");
437 }),
438 set: (request) =>
439 guarded(
440 request,
441 writeJson(resourceKey(request), request.value).pipe(Effect.as(request.value)),
442 ),
443 delete: (request) =>
444 guarded(
445 request,
446 use((o) => o.delete(resourceKey(request))),
447 ),
448 // A stack-wide delete locks every stage it is about to remove, so a
449 // concurrent deploy fails the lease instead of racing the delete.
450 deleteStack: ({ stack, stage }) =>
451 stage === undefined
452 ? verifyHeldLeases.pipe(
453 Effect.andThen(state.listStages(stack)),
454 Effect.flatMap((stages) =>
455 Effect.forEach(
456 stages,
457 (found) =>
458 guarded(
459 { stack, stage: found },
460 deleteAll(stagePrefix({ stack, stage: found })),
461 ),
462 { discard: true },
463 ),
464 ),
465 Effect.andThen(deleteAll(`${prefix}${stack}/`)),
466 )
467 : guarded({ stack, stage }, deleteAll(stagePrefix({ stack, stage }))),
468 list: (request) =>
469 guarded(
470 request,
471 use((o) => o.listKeys(stagePrefix(request))).pipe(
472 Effect.map((keys) =>
473 keys
474 .map((key) => key.slice(stagePrefix(request).length))
475 // Filter bookkeeping before decoding — `decodeFqn` replaces
476 // `__` with `/`, which would turn `__stack_output__` into
477 // `/stack_output/` and slip past the filter.
478 .filter(
479 (file) => file !== OUTPUT_FILE && file !== LEASE_FILE && file.endsWith(".json"),
480 )
481 .map((file) => decodeFqn(file.replace(/\.json$/, ""))),
482 ),
483 ),
484 ),
485 getOutput: (request) => guarded(request, readJson(outputKey(request))),
486 setOutput: (request) =>
487 guarded(
488 request,
489 writeJson(outputKey(request), request.value).pipe(Effect.as(request.value)),
490 ),
491 };
492 return state;
493 });