The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 04:45 UTC
| 1 | import * as Clock from "effect/Clock"; |
| 2 | import * as Duration from "effect/Duration"; |
| 3 | import * as Effect from "effect/Effect"; |
| 4 | import * as Schedule from "effect/Schedule"; |
| 5 | import * as Scope from "effect/Scope"; |
| 6 | import * as Semaphore from "effect/Semaphore"; |
| 7 | import { decodeFqn, encodeFqn } from "../FQN.ts"; |
| 8 | import { STATE_STORE_VERSION } from "./HttpStateApi.ts"; |
| 9 | import { StateStoreError, type PersistedState, type StateService } from "./State.ts"; |
| 10 | import { encodeState, reviveState } from "./StateEncoding.ts"; |
| 11 | |
| 12 | /** |
| 13 | * A state store over an object storage bucket (S3, Cloud Storage, …). |
| 14 | * |
| 15 | * The store is laid out exactly like the local state store's file tree, |
| 16 | * with the bucket (plus an optional prefix) in place of `.alchemy/state`: |
| 17 | * |
| 18 | * ``` |
| 19 | * {prefix}{stack}/{stage}/{fqn}.json |
| 20 | * {prefix}{stack}/{stage}/__stack_output__.json |
| 21 | * {prefix}{stack}/{stage}/__lease__.json |
| 22 | * ``` |
| 23 | * |
| 24 | * Concurrent deploys of one stack and stage share the `__lease__.json` |
| 25 | * object, written with conditional puts. The second deploy fails |
| 26 | * immediately instead of mixing rows. The holder refreshes the lease until |
| 27 | * the store closes; a killed process stops refreshing, the lease expires, |
| 28 | * and the next deploy can take it. |
| 29 | * |
| 30 | * Each backend supplies a {@link BucketObjects}: plain object operations |
| 31 | * plus a version token (an S3 ETag, a Cloud Storage generation) for the |
| 32 | * conditional writes the lease needs. |
| 33 | */ |
| 34 | |
| 35 | /** The bookkeeping object that stores a stack's resolved output. */ |
| 36 | const OUTPUT_FILE = "__stack_output__.json"; |
| 37 | |
| 38 | /** |
| 39 | * Per `(stack, stage)` lease. A delete of the stage leaves it in place so |
| 40 | * the lock outlives the objects it was guarding; the store deletes it on |
| 41 | * close. |
| 42 | */ |
| 43 | /** The lease token `alchemy state migrate` leaves on a stage it moved to DynamoDB. */ |
| 44 | const MIGRATED_TOKEN = "migrated:dynamodb"; |
| 45 | |
| 46 | const LEASE_FILE = "__lease__.json"; |
| 47 | |
| 48 | /** How long a lease stays valid without a refresh. */ |
| 49 | const LEASE_TTL_MS = 60_000; |
| 50 | |
| 51 | /** How long a passing lease check is trusted. A failure is never cached. */ |
| 52 | const LEASE_CHECK_TTL_MS = 5_000; |
| 53 | |
| 54 | /** A write condition, in terms of the backend's version token. */ |
| 55 | export type WriteCondition = |
| 56 | /** Only create the object; fail if it exists. */ |
| 57 | | { readonly ifAbsent: true } |
| 58 | /** Only overwrite the object at exactly this version. */ |
| 59 | | { readonly ifVersion: string }; |
| 60 | |
| 61 | /** |
| 62 | * The object operations a bucket backend implements. Every operation is |
| 63 | * self-contained (`R = never`) and fails with {@link StateStoreError}. |
| 64 | */ |
| 65 | export interface BucketObjects { |
| 66 | /** Every key under `prefix`, across pagination. */ |
| 67 | readonly listKeys: (prefix: string) => Effect.Effect<ReadonlyArray<string>, StateStoreError>; |
| 68 | /** |
| 69 | * The immediate "directory" names under `prefix` (delimiter `/`), without |
| 70 | * the prefix or the trailing slash. |
| 71 | */ |
| 72 | readonly listChildren: (prefix: string) => Effect.Effect<ReadonlyArray<string>, StateStoreError>; |
| 73 | /** An object's text and version, or `undefined` when it doesn't exist. */ |
| 74 | readonly read: ( |
| 75 | key: string, |
| 76 | ) => Effect.Effect< |
| 77 | { readonly text: string; readonly version: string } | undefined, |
| 78 | StateStoreError |
| 79 | >; |
| 80 | /** |
| 81 | * Write an object. Returns its new version, or `undefined` when |
| 82 | * `condition` did not hold. |
| 83 | */ |
| 84 | readonly write: ( |
| 85 | key: string, |
| 86 | body: string, |
| 87 | condition?: WriteCondition, |
| 88 | ) => Effect.Effect<string | undefined, StateStoreError>; |
| 89 | /** |
| 90 | * Delete an object, only at `ifVersion` when given. Deleting a missing |
| 91 | * object, or one at another version, succeeds without deleting. |
| 92 | */ |
| 93 | readonly delete: (key: string, ifVersion?: string) => Effect.Effect<void, StateStoreError>; |
| 94 | /** Delete many objects. Missing objects are ignored. */ |
| 95 | readonly deleteMany: (keys: ReadonlyArray<string>) => Effect.Effect<void, StateStoreError>; |
| 96 | } |
| 97 | |
| 98 | export interface BucketStateOptions { |
| 99 | /** The store's id, e.g. `"s3"` or `"gcs"`. */ |
| 100 | readonly id: string; |
| 101 | /** How errors name the store, e.g. `"S3"` or `"Cloud Storage"`. */ |
| 102 | readonly label: string; |
| 103 | /** Key prefix within the bucket. A trailing `/` is added. */ |
| 104 | readonly prefix?: string; |
| 105 | /** |
| 106 | * The bucket's objects. Runs once, on the first state operation, and is |
| 107 | * cached: ensure the bucket exists here, never at construction. |
| 108 | */ |
| 109 | readonly objects: Effect.Effect<BucketObjects, StateStoreError>; |
| 110 | } |
| 111 | |
| 112 | /** |
| 113 | * Tokens of the leases this process holds, so a conflict with one of them |
| 114 | * can say the holder is an earlier, still-open build in the same process. |
| 115 | */ |
| 116 | const heldInProcess = new Set<string>(); |
| 117 | |
| 118 | interface LeaseRecord { |
| 119 | readonly token: string; |
| 120 | readonly expiresAt: number; |
| 121 | } |
| 122 | |
| 123 | interface HeldLease { |
| 124 | readonly checkLive: Effect.Effect<void, StateStoreError>; |
| 125 | } |
| 126 | |
| 127 | /** |
| 128 | * A passing check is trusted for `ttlMs`, so a burst of operations does |
| 129 | * one round-trip. A failing check is never cached. |
| 130 | */ |
| 131 | const amortizeCheck = ( |
| 132 | checkLive: Effect.Effect<void, StateStoreError>, |
| 133 | ttlMs: number, |
| 134 | ): Effect.Effect<void, StateStoreError> => { |
| 135 | let lastOkAt: number | undefined; |
| 136 | return Effect.suspend(() => { |
| 137 | if (lastOkAt !== undefined && Date.now() - lastOkAt < ttlMs) { |
| 138 | return Effect.void; |
| 139 | } |
| 140 | return checkLive.pipe( |
| 141 | Effect.tap(() => |
| 142 | Effect.sync(() => { |
| 143 | lastOkAt = Date.now(); |
| 144 | }), |
| 145 | ), |
| 146 | ); |
| 147 | }); |
| 148 | }; |
| 149 | |
| 150 | const parseLease = (text: string): LeaseRecord => { |
| 151 | let parsed: unknown; |
| 152 | try { |
| 153 | parsed = JSON.parse(text); |
| 154 | } catch { |
| 155 | parsed = undefined; |
| 156 | } |
| 157 | return typeof parsed === "object" && |
| 158 | parsed !== null && |
| 159 | "token" in parsed && |
| 160 | "expiresAt" in parsed && |
| 161 | typeof parsed.token === "string" && |
| 162 | typeof parsed.expiresAt === "number" |
| 163 | ? { token: parsed.token, expiresAt: parsed.expiresAt } |
| 164 | : { token: "", expiresAt: 0 }; |
| 165 | }; |
| 166 | |
| 167 | /** |
| 168 | * Construct a bucket-backed {@link StateService}. Leases are bound to the |
| 169 | * ambient scope; callers that need them to outlive the first operation |
| 170 | * bind this effect to a longer scope with `Scope.provide`. |
| 171 | */ |
| 172 | export const makeBucketState = (options: BucketStateOptions) => |
| 173 | Effect.gen(function* () { |
| 174 | const leaseScope = yield* Effect.scope; |
| 175 | const prefix = options.prefix ? `${options.prefix.replace(/\/+$/, "")}/` : ""; |
| 176 | const objects = yield* Effect.cached(options.objects); |
| 177 | const use = <A>( |
| 178 | f: (objects: BucketObjects) => Effect.Effect<A, StateStoreError>, |
| 179 | ): Effect.Effect<A, StateStoreError> => objects.pipe(Effect.flatMap(f)); |
| 180 | |
| 181 | const stagePrefix = ({ stack, stage }: { stack: string; stage: string }) => |
| 182 | `${prefix}${stack}/${stage}/`; |
| 183 | const resourceKey = (request: { stack: string; stage: string; fqn: string }) => |
| 184 | `${stagePrefix(request)}${encodeFqn(request.fqn)}.json`; |
| 185 | const outputKey = (request: { stack: string; stage: string }) => |
| 186 | `${stagePrefix(request)}${OUTPUT_FILE}`; |
| 187 | const leaseKey = (request: { stack: string; stage: string }) => |
| 188 | `${stagePrefix(request)}${LEASE_FILE}`; |
| 189 | |
| 190 | const readJson = <T>(key: string) => |
| 191 | use((o) => o.read(key)).pipe( |
| 192 | Effect.flatMap((object) => |
| 193 | object === undefined |
| 194 | ? Effect.undefined |
| 195 | : Effect.try({ |
| 196 | try: () => JSON.parse(object.text, reviveState) as T, |
| 197 | catch: (cause) => |
| 198 | new StateStoreError({ |
| 199 | message: `Failed to parse state object '${key}'`, |
| 200 | cause: cause instanceof Error ? cause : undefined, |
| 201 | }), |
| 202 | }), |
| 203 | ), |
| 204 | ); |
| 205 | |
| 206 | const writeJson = (key: string, value: unknown) => |
| 207 | use((o) => o.write(key, JSON.stringify(encodeState(value), null, 2))); |
| 208 | |
| 209 | /** Delete every object under `keyPrefix`, except leases. Idempotent. */ |
| 210 | const deleteAll = (keyPrefix: string) => |
| 211 | use((o) => |
| 212 | o.listKeys(keyPrefix).pipe( |
| 213 | // Leave the lease object. Releasing it here would open the stage |
| 214 | // before this delete finishes, and the store still holds the lock. |
| 215 | Effect.map((keys) => keys.filter((key) => !key.endsWith(`/${LEASE_FILE}`))), |
| 216 | Effect.flatMap((keys) => (keys.length === 0 ? Effect.void : o.deleteMany(keys))), |
| 217 | ), |
| 218 | ); |
| 219 | |
| 220 | const heldError = (stack: string, stage: string, holder: string | undefined) => |
| 221 | // The message always starts "another deploy holds": callers match on it |
| 222 | // (the AWS migration tells a held stage from a failed one by it). |
| 223 | new StateStoreError({ |
| 224 | message: |
| 225 | holder !== undefined && heldInProcess.has(holder) |
| 226 | ? `another deploy holds the ${options.label} state lock '${stack}/${stage}': this process holds it, from an earlier operation on this stage that is still open. Run each operation in its own Effect.scoped so its build closes first.` |
| 227 | : `another deploy holds the ${options.label} state lock '${stack}/${stage}'`, |
| 228 | }); |
| 229 | |
| 230 | const lostError = (stack: string, stage: string) => |
| 231 | new StateStoreError({ |
| 232 | message: `the ${options.label} state lock '${stack}/${stage}' was lost mid-run; refusing to continue unlocked`, |
| 233 | }); |
| 234 | |
| 235 | const readLease = (key: string) => |
| 236 | use((o) => o.read(key)).pipe( |
| 237 | Effect.map((object) => |
| 238 | object === undefined |
| 239 | ? undefined |
| 240 | : { version: object.version, record: parseLease(object.text) }, |
| 241 | ), |
| 242 | ); |
| 243 | |
| 244 | const putLease = (key: string, record: LeaseRecord, condition: WriteCondition) => |
| 245 | use((o) => o.write(key, JSON.stringify(record), condition)); |
| 246 | |
| 247 | const acquireLease = (stack: string, stage: string) => |
| 248 | Effect.gen(function* () { |
| 249 | const key = leaseKey({ stack, stage }); |
| 250 | const token = yield* Effect.sync(() => crypto.randomUUID()); |
| 251 | const now = yield* Clock.currentTimeMillis; |
| 252 | const fresh: LeaseRecord = { token, expiresAt: now + LEASE_TTL_MS }; |
| 253 | let version = yield* putLease(key, fresh, { ifAbsent: true }); |
| 254 | if (version === undefined) { |
| 255 | const existing = yield* readLease(key); |
| 256 | const stillHeld = existing === undefined || existing.record.expiresAt > now; |
| 257 | if (existing?.record.token === MIGRATED_TOKEN) { |
| 258 | return yield* new StateStoreError({ |
| 259 | message: `${heldError(stack, stage, undefined).message}: the stage moved to a DynamoDB state store. Deploy it with a version of alchemy that has AWS.state() on DynamoDB, or run 'alchemy state migrate --rollback' with that version to move it back.`, |
| 260 | }); |
| 261 | } |
| 262 | if (stillHeld) { |
| 263 | return yield* heldError(stack, stage, existing?.record.token); |
| 264 | } |
| 265 | const replaced = yield* putLease(key, fresh, { ifVersion: existing.version }); |
| 266 | if (replaced === undefined) { |
| 267 | return yield* heldError(stack, stage, undefined); |
| 268 | } |
| 269 | version = replaced; |
| 270 | } |
| 271 | |
| 272 | heldInProcess.add(token); |
| 273 | const cursor = { version, lost: false }; |
| 274 | const gate = Semaphore.makeUnsafe(1); |
| 275 | |
| 276 | const renew = Semaphore.withPermits( |
| 277 | gate, |
| 278 | 1, |
| 279 | )( |
| 280 | Effect.gen(function* () { |
| 281 | if (cursor.lost) return; |
| 282 | const renewedAt = yield* Clock.currentTimeMillis; |
| 283 | const next = yield* putLease( |
| 284 | key, |
| 285 | { token, expiresAt: renewedAt + LEASE_TTL_MS }, |
| 286 | { ifVersion: cursor.version }, |
| 287 | ); |
| 288 | if (next === undefined) cursor.lost = true; |
| 289 | else cursor.version = next; |
| 290 | }), |
| 291 | ).pipe( |
| 292 | Effect.retry({ |
| 293 | while: () => !cursor.lost, |
| 294 | schedule: Schedule.spaced("1 second"), |
| 295 | times: 3, |
| 296 | }), |
| 297 | Effect.catch(() => |
| 298 | Effect.sync(() => { |
| 299 | cursor.lost = true; |
| 300 | }), |
| 301 | ), |
| 302 | ); |
| 303 | |
| 304 | const verify = Semaphore.withPermits( |
| 305 | gate, |
| 306 | 1, |
| 307 | )( |
| 308 | Effect.gen(function* () { |
| 309 | if (cursor.lost) return yield* lostError(stack, stage); |
| 310 | const existing = yield* readLease(key); |
| 311 | if (existing === undefined || existing.record.token !== token) { |
| 312 | cursor.lost = true; |
| 313 | return yield* lostError(stack, stage); |
| 314 | } |
| 315 | const observedAt = yield* Clock.currentTimeMillis; |
| 316 | if (existing.record.expiresAt <= observedAt) { |
| 317 | // The lease lapsed without anyone taking it: a refresh queued |
| 318 | // behind a large deploy's writes missed its window. Take it |
| 319 | // back at the version we read, so a deploy that took it in |
| 320 | // the meantime wins and this one stops. |
| 321 | const reclaimed = yield* putLease( |
| 322 | key, |
| 323 | { token, expiresAt: observedAt + LEASE_TTL_MS }, |
| 324 | { ifVersion: existing.version }, |
| 325 | ); |
| 326 | if (reclaimed === undefined) { |
| 327 | cursor.lost = true; |
| 328 | return yield* lostError(stack, stage); |
| 329 | } |
| 330 | cursor.version = reclaimed; |
| 331 | return; |
| 332 | } |
| 333 | cursor.version = existing.version; |
| 334 | }), |
| 335 | ); |
| 336 | |
| 337 | // Interrupt the refresh before releasing, so a refresh cannot |
| 338 | // land after the lease has been released. |
| 339 | yield* Scope.addFinalizer( |
| 340 | leaseScope, |
| 341 | Semaphore.withPermits( |
| 342 | gate, |
| 343 | 1, |
| 344 | )( |
| 345 | Effect.gen(function* () { |
| 346 | cursor.lost = true; |
| 347 | heldInProcess.delete(token); |
| 348 | // Read the version we actually hold. A refresh that landed |
| 349 | // after the cursor was last stored would make a release |
| 350 | // against the old version a no-op, and the lease would stay. |
| 351 | const existing = yield* readLease(key); |
| 352 | if (existing === undefined || existing.record.token !== token) return; |
| 353 | // Delete rather than expire: a leftover lease object would keep |
| 354 | // a destroyed stage listed by `listStages`. |
| 355 | yield* use((o) => o.delete(key, existing.version)); |
| 356 | }).pipe(Effect.ignore), |
| 357 | ), |
| 358 | ); |
| 359 | yield* renew.pipe( |
| 360 | Effect.delay(Duration.millis(Math.floor(LEASE_TTL_MS / 3))), |
| 361 | Effect.forever, |
| 362 | Effect.interruptible, |
| 363 | Effect.forkIn(leaseScope), |
| 364 | ); |
| 365 | |
| 366 | return { checkLive: amortizeCheck(verify, LEASE_CHECK_TTL_MS) } satisfies HeldLease; |
| 367 | }); |
| 368 | |
| 369 | // One lease per (stack, stage), taken on the first operation and held |
| 370 | // for the store's lifetime. The mutex keeps two concurrent first |
| 371 | // operations in this process from contending with each other. |
| 372 | const leaseMutex = Semaphore.makeUnsafe(1); |
| 373 | const leases = new Map<string, Effect.Effect<HeldLease, StateStoreError>>(); |
| 374 | |
| 375 | const leaseFor = (stack: string, stage: string): Effect.Effect<HeldLease, StateStoreError> => |
| 376 | Semaphore.withPermits( |
| 377 | leaseMutex, |
| 378 | 1, |
| 379 | )( |
| 380 | Effect.gen(function* () { |
| 381 | const key = `${stack}/${stage}`; |
| 382 | const existing = leases.get(key); |
| 383 | if (existing !== undefined) return existing; |
| 384 | // Drop a failed acquire. Caching it would make the next operation |
| 385 | // report the same holder after that holder has gone. |
| 386 | const cached = yield* Effect.cached( |
| 387 | acquireLease(stack, stage).pipe( |
| 388 | Effect.tapError(() => |
| 389 | Effect.sync(() => { |
| 390 | if (leases.get(key) === cached) { |
| 391 | leases.delete(key); |
| 392 | } |
| 393 | }), |
| 394 | ), |
| 395 | ), |
| 396 | ); |
| 397 | leases.set(key, cached); |
| 398 | return cached; |
| 399 | }), |
| 400 | ).pipe(Effect.flatMap((lease) => lease)); |
| 401 | |
| 402 | const guarded = <A>( |
| 403 | request: { stack: string; stage: string }, |
| 404 | op: Effect.Effect<A, StateStoreError>, |
| 405 | ): Effect.Effect<A, StateStoreError> => |
| 406 | leaseFor(request.stack, request.stage).pipe( |
| 407 | Effect.flatMap((lease) => lease.checkLive), |
| 408 | Effect.andThen(op), |
| 409 | ); |
| 410 | |
| 411 | const verifyHeldLeases: Effect.Effect<void, StateStoreError> = Effect.suspend(() => |
| 412 | Effect.forEach( |
| 413 | Array.from(leases.values()), |
| 414 | (lease) => lease.pipe(Effect.flatMap((held) => held.checkLive)), |
| 415 | { discard: true }, |
| 416 | ), |
| 417 | ); |
| 418 | |
| 419 | const listChildren = (keyPrefix: string) => |
| 420 | use((o) => o.listChildren(keyPrefix)).pipe(Effect.map((names) => Array.from(names))); |
| 421 | |
| 422 | const state: StateService = { |
| 423 | id: options.id, |
| 424 | getVersion: () => Effect.succeed(STATE_STORE_VERSION), |
| 425 | listStacks: () => verifyHeldLeases.pipe(Effect.andThen(listChildren(prefix))), |
| 426 | listStages: (stack: string) => |
| 427 | verifyHeldLeases.pipe(Effect.andThen(listChildren(`${prefix}${stack}/`))), |
| 428 | get: (request) => guarded(request, readJson<PersistedState>(resourceKey(request))), |
| 429 | getReplacedResources: Effect.fn(function* (request) { |
| 430 | return (yield* Effect.forEach(yield* state.list(request), (fqn) => |
| 431 | state.get({ |
| 432 | stack: request.stack, |
| 433 | stage: request.stage, |
| 434 | fqn, |
| 435 | }), |
| 436 | )).filter((r) => r?.status === "replaced"); |
| 437 | }), |
| 438 | set: (request) => |
| 439 | guarded( |
| 440 | request, |
| 441 | writeJson(resourceKey(request), request.value).pipe(Effect.as(request.value)), |
| 442 | ), |
| 443 | delete: (request) => |
| 444 | guarded( |
| 445 | request, |
| 446 | use((o) => o.delete(resourceKey(request))), |
| 447 | ), |
| 448 | // A stack-wide delete locks every stage it is about to remove, so a |
| 449 | // concurrent deploy fails the lease instead of racing the delete. |
| 450 | deleteStack: ({ stack, stage }) => |
| 451 | stage === undefined |
| 452 | ? verifyHeldLeases.pipe( |
| 453 | Effect.andThen(state.listStages(stack)), |
| 454 | Effect.flatMap((stages) => |
| 455 | Effect.forEach( |
| 456 | stages, |
| 457 | (found) => |
| 458 | guarded( |
| 459 | { stack, stage: found }, |
| 460 | deleteAll(stagePrefix({ stack, stage: found })), |
| 461 | ), |
| 462 | { discard: true }, |
| 463 | ), |
| 464 | ), |
| 465 | Effect.andThen(deleteAll(`${prefix}${stack}/`)), |
| 466 | ) |
| 467 | : guarded({ stack, stage }, deleteAll(stagePrefix({ stack, stage }))), |
| 468 | list: (request) => |
| 469 | guarded( |
| 470 | request, |
| 471 | use((o) => o.listKeys(stagePrefix(request))).pipe( |
| 472 | Effect.map((keys) => |
| 473 | keys |
| 474 | .map((key) => key.slice(stagePrefix(request).length)) |
| 475 | // Filter bookkeeping before decoding — `decodeFqn` replaces |
| 476 | // `__` with `/`, which would turn `__stack_output__` into |
| 477 | // `/stack_output/` and slip past the filter. |
| 478 | .filter( |
| 479 | (file) => file !== OUTPUT_FILE && file !== LEASE_FILE && file.endsWith(".json"), |
| 480 | ) |
| 481 | .map((file) => decodeFqn(file.replace(/\.json$/, ""))), |
| 482 | ), |
| 483 | ), |
| 484 | ), |
| 485 | getOutput: (request) => guarded(request, readJson(outputKey(request))), |
| 486 | setOutput: (request) => |
| 487 | guarded( |
| 488 | request, |
| 489 | writeJson(outputKey(request), request.value).pipe(Effect.as(request.value)), |
| 490 | ), |
| 491 | }; |
| 492 | return state; |
| 493 | }); |