The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import { |
| 2 | BadGateway, |
| 3 | GatewayTimeout, |
| 4 | InternalServerError, |
| 5 | ServiceUnavailable, |
| 6 | TooManyRequests, |
| 7 | } from "@distilled.cloud/fly-io/Errors"; |
| 8 | import * as machines from "@distilled.cloud/fly-io/machines"; |
| 9 | import * as Retry from "@distilled.cloud/fly-io/Retry"; |
| 10 | import * as Clock from "effect/Clock"; |
| 11 | import * as Data from "effect/Data"; |
| 12 | import * as Deferred from "effect/Deferred"; |
| 13 | import * as Duration from "effect/Duration"; |
| 14 | import * as Effect from "effect/Effect"; |
| 15 | import * as Fiber from "effect/Fiber"; |
| 16 | import * as FetchHttpClient from "effect/http/FetchHttpClient"; |
| 17 | import * as HttpClient from "effect/http/HttpClient"; |
| 18 | import * as HttpClientRequest from "effect/http/HttpClientRequest"; |
| 19 | import * as Option from "effect/Option"; |
| 20 | import * as Schedule from "effect/Schedule"; |
| 21 | |
| 22 | const TTL_SECONDS = 120; |
| 23 | const EXPIRY_MARGIN_MS = 15_000; |
| 24 | |
| 25 | const retrySchedule = <E>(): Schedule.Schedule<Duration.Duration, E> => |
| 26 | Schedule.exponential("500 millis").pipe( |
| 27 | Schedule.modifyDelay<Duration.Duration, E>(({ input, duration }) => |
| 28 | Effect.succeed( |
| 29 | (TooManyRequests.is(input) || |
| 30 | InternalServerError.is(input) || |
| 31 | BadGateway.is(input) || |
| 32 | ServiceUnavailable.is(input) || |
| 33 | GatewayTimeout.is(input)) && |
| 34 | input.retryAfter |
| 35 | ? Math.min(Duration.toMillis(input.retryAfter), 60_000) |
| 36 | : Math.min(Duration.toMillis(duration), 5_000), |
| 37 | ), |
| 38 | ), |
| 39 | Schedule.addDelay<Duration.Duration, E>(() => |
| 40 | Effect.sync(() => Duration.millis(Math.random() * 50)), |
| 41 | ), |
| 42 | ); |
| 43 | |
| 44 | export class MachineLeaseBusy extends Data.TaggedError("Fly.MachineLeaseBusy")<{ |
| 45 | appName: string; |
| 46 | machineId: string; |
| 47 | }> {} |
| 48 | |
| 49 | export class MachineLeaseLost extends Data.TaggedError("Fly.MachineLeaseLost")<{ |
| 50 | appName: string; |
| 51 | machineId: string; |
| 52 | reason: string; |
| 53 | }> { |
| 54 | get message() { |
| 55 | return `Machine lease authority lost for ${this.appName}/${this.machineId}: ${this.reason}`; |
| 56 | } |
| 57 | } |
| 58 | |
| 59 | export class MachineMutationUncertain extends Data.TaggedError("Fly.MachineMutationUncertain")<{ |
| 60 | appName: string; |
| 61 | machineId: string; |
| 62 | }> {} |
| 63 | |
| 64 | interface HeldLease { |
| 65 | nonce: string; |
| 66 | deadline: number; |
| 67 | renewAt: number; |
| 68 | } |
| 69 | |
| 70 | export const makeMachineLeases = Effect.fn(function* (appName: string) { |
| 71 | const scope = yield* Effect.scope; |
| 72 | const held = new Map<string, HeldLease>(); |
| 73 | const deleting = new Set<string>(); |
| 74 | const renewals = new Set<Fiber.Fiber<void, never>>(); |
| 75 | const uncertain = new Set<string>(); |
| 76 | const lost = yield* Deferred.make<never, MachineLeaseLost>(); |
| 77 | let failure: MachineLeaseLost | undefined; |
| 78 | const lose = (machineId: string, reason: string) => |
| 79 | Effect.gen(function* () { |
| 80 | failure ??= new MachineLeaseLost({ appName, machineId, reason }); |
| 81 | yield* Deferred.fail(lost, failure); |
| 82 | return yield* failure; |
| 83 | }); |
| 84 | const check = Effect.gen(function* () { |
| 85 | if (failure) return yield* failure; |
| 86 | const now = yield* Clock.currentTimeMillis; |
| 87 | for (const [machineId, lease] of held) { |
| 88 | if (now >= lease.deadline) return yield* lose(machineId, "lease authority expired"); |
| 89 | } |
| 90 | }); |
| 91 | const guard = <A, E, R>(effect: Effect.Effect<A, E, R>) => |
| 92 | check.pipe(Effect.andThen(Effect.raceFirst(effect, Deferred.await(lost)))); |
| 93 | const record = (machineId: string, response: machines.MachineLease, requestedAt: number) => |
| 94 | Effect.gen(function* () { |
| 95 | const lease = response.data; |
| 96 | const now = yield* Clock.currentTimeMillis; |
| 97 | const deadline = |
| 98 | Math.min((lease?.expires_at ?? 0) * 1000, requestedAt + TTL_SECONDS * 1000) - |
| 99 | EXPIRY_MARGIN_MS; |
| 100 | const captured = lease?.nonce |
| 101 | ? { nonce: lease.nonce, deadline, renewAt: requestedAt + 25_000 } |
| 102 | : undefined; |
| 103 | if (captured) held.set(machineId, captured); |
| 104 | if (!captured || !Number.isFinite(deadline) || deadline <= now) { |
| 105 | return yield* lose(machineId, "invalid or expired lease response"); |
| 106 | } |
| 107 | return captured; |
| 108 | }); |
| 109 | const acquire = (machineIds: readonly string[]) => |
| 110 | Effect.gen(function* () { |
| 111 | for (const machineId of [...new Set(machineIds)].sort()) { |
| 112 | yield* check; |
| 113 | if (held.has(machineId)) continue; |
| 114 | // Save a returned nonce atomically; uncertain interrupted requests expire at Fly. |
| 115 | yield* Effect.uninterruptibleMask((restore) => |
| 116 | Effect.gen(function* () { |
| 117 | const requestedAt = yield* Clock.currentTimeMillis; |
| 118 | const response = yield* restore( |
| 119 | machines |
| 120 | .createMachineLease({ |
| 121 | app_name: appName, |
| 122 | machine_id: machineId, |
| 123 | ttl: TTL_SECONDS, |
| 124 | description: "Alchemy Machine lifecycle", |
| 125 | }) |
| 126 | .pipe( |
| 127 | Retry.none, |
| 128 | Effect.retry({ |
| 129 | times: 8, |
| 130 | schedule: retrySchedule<machines.CreateMachineLeaseError>(), |
| 131 | while: (error) => error._tag === "Conflict" || error._tag === "TooManyRequests", |
| 132 | }), |
| 133 | Effect.timeout("30 seconds"), |
| 134 | Effect.catchTag("Conflict", () => |
| 135 | Effect.fail(new MachineLeaseBusy({ appName, machineId })), |
| 136 | ), |
| 137 | Effect.catchTag(["HttpClientError", "GatewayTimeout", "TimeoutError"], (error) => |
| 138 | lose(machineId, `acquisition uncertain: ${error._tag}`), |
| 139 | ), |
| 140 | ), |
| 141 | ); |
| 142 | const lease = yield* record(machineId, response, requestedAt); |
| 143 | const renewal = yield* heartbeat(machineId, lease).pipe( |
| 144 | Effect.interruptible, |
| 145 | Effect.forkIn(scope), |
| 146 | ); |
| 147 | renewals.add(renewal); |
| 148 | }), |
| 149 | ); |
| 150 | yield* check; |
| 151 | } |
| 152 | }); |
| 153 | const nonce = (machineId: string) => |
| 154 | Effect.gen(function* () { |
| 155 | yield* check; |
| 156 | const lease = held.get(machineId); |
| 157 | if (!lease) return yield* lose(machineId, "target is outside the acquired lease set"); |
| 158 | return lease.nonce; |
| 159 | }); |
| 160 | const mutate = <A, E extends { readonly _tag: string }, R>( |
| 161 | machineId: string, |
| 162 | operation: (nonce: string) => Effect.Effect<A, E, R>, |
| 163 | options: { idempotent?: boolean; timeoutMs?: number } = {}, |
| 164 | ) => |
| 165 | guard( |
| 166 | Effect.gen(function* () { |
| 167 | const value = yield* nonce(machineId); |
| 168 | return yield* operation(value).pipe(Retry.none); |
| 169 | }).pipe( |
| 170 | Effect.retry({ |
| 171 | times: 8, |
| 172 | schedule: retrySchedule<E | MachineLeaseLost>(), |
| 173 | // Only explicit throttling is safe to replay for update/stop/delete. |
| 174 | while: (error) => |
| 175 | error._tag === "TooManyRequests" || |
| 176 | (options.idempotent === true && |
| 177 | (error._tag === "InternalServerError" || |
| 178 | error._tag === "BadGateway" || |
| 179 | error._tag === "ServiceUnavailable" || |
| 180 | error._tag === "GatewayTimeout")), |
| 181 | }), |
| 182 | Effect.timeout(options.timeoutMs ?? 120_000), |
| 183 | Effect.catch((error) => |
| 184 | Effect.gen(function* () { |
| 185 | if (error._tag === "Conflict" || error._tag === "Forbidden") { |
| 186 | return yield* lose(machineId, `leased mutation rejected: ${error._tag}`); |
| 187 | } |
| 188 | // Transport errors retain the request, including its private nonce header. |
| 189 | if ( |
| 190 | error._tag === "HttpClientError" || |
| 191 | error._tag === "GatewayTimeout" || |
| 192 | error._tag === "TimeoutError" |
| 193 | ) { |
| 194 | return yield* new MachineMutationUncertain({ |
| 195 | appName, |
| 196 | machineId, |
| 197 | }); |
| 198 | } |
| 199 | return yield* Effect.fail(error); |
| 200 | }), |
| 201 | ), |
| 202 | ), |
| 203 | ); |
| 204 | const forget = (machineId: string) => |
| 205 | Effect.sync(() => { |
| 206 | held.delete(machineId); |
| 207 | }); |
| 208 | const confirmRemoved = (machineId: string) => |
| 209 | guard( |
| 210 | machines |
| 211 | .getMachine({ |
| 212 | app_name: appName, |
| 213 | machine_id: machineId, |
| 214 | }) |
| 215 | .pipe( |
| 216 | Retry.none, |
| 217 | Effect.map((machine) => machine.id === machineId && machine.state === "destroyed"), |
| 218 | Effect.catchTag("NotFound", () => Effect.succeed(true)), |
| 219 | Effect.catchTag( |
| 220 | [ |
| 221 | "TooManyRequests", |
| 222 | "InternalServerError", |
| 223 | "BadGateway", |
| 224 | "ServiceUnavailable", |
| 225 | "GatewayTimeout", |
| 226 | ], |
| 227 | () => Effect.succeed(false), |
| 228 | ), |
| 229 | Effect.catchReason("HttpClientError", "TransportError", () => Effect.succeed(false)), |
| 230 | Effect.repeat({ |
| 231 | times: 8, |
| 232 | schedule: Schedule.spaced("2 seconds"), |
| 233 | until: (absent) => absent, |
| 234 | }), |
| 235 | Effect.timeoutOrElse({ duration: "30 seconds", orElse: () => Effect.succeed(false) }), |
| 236 | Effect.catch((error) => |
| 237 | Effect.gen(function* () { |
| 238 | if (!held.has(machineId)) return true; |
| 239 | if (error._tag === "Forbidden") { |
| 240 | return yield* lose(machineId, "removal observation forbidden"); |
| 241 | } |
| 242 | return yield* error; |
| 243 | }), |
| 244 | ), |
| 245 | // Another observer's confirmed removal wins over a late uncertain response. |
| 246 | Effect.map((absent) => absent || !held.has(machineId)), |
| 247 | Effect.tap((absent) => (absent ? forget(machineId) : Effect.void)), |
| 248 | ), |
| 249 | ); |
| 250 | const remove = <A, E extends { readonly _tag: string }, R>( |
| 251 | machineId: string, |
| 252 | operation: (nonce: string) => Effect.Effect<A, E, R>, |
| 253 | ) => |
| 254 | Effect.gen(function* () { |
| 255 | const http = yield* HttpClient.HttpClient; |
| 256 | const fetchOptions = yield* Effect.serviceOption(FetchHttpClient.RequestInit); |
| 257 | yield* Effect.sync(() => { |
| 258 | deleting.add(machineId); |
| 259 | }); |
| 260 | yield* mutate(machineId, (nonce) => |
| 261 | operation(nonce).pipe( |
| 262 | // Bun can replay DELETE on a reused socket below the SDK retry policy. |
| 263 | Effect.provideService( |
| 264 | HttpClient.HttpClient, |
| 265 | HttpClient.mapRequest(http, HttpClientRequest.setHeader("connection", "close")), |
| 266 | ), |
| 267 | Effect.provideService(FetchHttpClient.RequestInit, { |
| 268 | ...Option.getOrUndefined(fetchOptions), |
| 269 | keepalive: false, |
| 270 | redirect: "error", |
| 271 | }), |
| 272 | ), |
| 273 | ).pipe( |
| 274 | Effect.asVoid, |
| 275 | Effect.catch((error) => |
| 276 | Effect.gen(function* () { |
| 277 | if (error._tag === "NotFound") return yield* forget(machineId); |
| 278 | if ( |
| 279 | error._tag === "Fly.MachineMutationUncertain" || |
| 280 | error._tag === "InternalServerError" || |
| 281 | error._tag === "BadGateway" || |
| 282 | error._tag === "ServiceUnavailable" |
| 283 | ) { |
| 284 | // Keep renewal aware of the removal until its exact target has been observed. |
| 285 | if (yield* confirmRemoved(machineId)) return; |
| 286 | } |
| 287 | return yield* Effect.fail(error); |
| 288 | }), |
| 289 | ), |
| 290 | ); |
| 291 | yield* forget(machineId); |
| 292 | }).pipe( |
| 293 | Effect.ensuring( |
| 294 | Effect.sync(() => { |
| 295 | deleting.delete(machineId); |
| 296 | }), |
| 297 | ), |
| 298 | ); |
| 299 | const nonceIfHeld = (machineId: string) => |
| 300 | check.pipe(Effect.map(() => held.get(machineId)?.nonce)); |
| 301 | |
| 302 | yield* Effect.addFinalizer(() => |
| 303 | Effect.gen(function* () { |
| 304 | // A late refresh can recreate a lease after a successful release. |
| 305 | yield* Fiber.interruptAll(renewals); |
| 306 | yield* Effect.forEach( |
| 307 | [...held.entries()] |
| 308 | .filter(([machineId]) => !uncertain.has(machineId)) |
| 309 | .sort(([a], [b]) => a.localeCompare(b)), |
| 310 | ([machineId, lease]) => |
| 311 | machines |
| 312 | .machinesReleaseLease({ |
| 313 | app_name: appName, |
| 314 | machine_id: machineId, |
| 315 | lease_nonce: lease.nonce, |
| 316 | }) |
| 317 | .pipe( |
| 318 | Retry.none, |
| 319 | Effect.timeout("5 seconds"), |
| 320 | Effect.interruptible, |
| 321 | Effect.catchTag("NotFound", () => Effect.void), |
| 322 | Effect.catch((error) => |
| 323 | Effect.logWarning( |
| 324 | "Fly lease release was not confirmed; expiry remains the safety bound", |
| 325 | { appName, machineId, error: error._tag }, |
| 326 | ), |
| 327 | ), |
| 328 | Effect.asVoid, |
| 329 | ), |
| 330 | { concurrency: 4 }, |
| 331 | ); |
| 332 | if (uncertain.size > 0) |
| 333 | yield* Effect.logWarning( |
| 334 | "Fly lease renewal was not confirmed; expiry remains the safety bound", |
| 335 | { appName, machineIds: [...uncertain] }, |
| 336 | ); |
| 337 | }), |
| 338 | ); |
| 339 | |
| 340 | const refresh = (machineId: string, lease: HeldLease) => |
| 341 | Effect.gen(function* () { |
| 342 | yield* check; |
| 343 | if (held.get(machineId) !== lease) return; |
| 344 | const requestedAt = yield* Clock.currentTimeMillis; |
| 345 | const response = yield* machines |
| 346 | .createMachineLease({ |
| 347 | app_name: appName, |
| 348 | machine_id: machineId, |
| 349 | ttl: TTL_SECONDS, |
| 350 | lease_nonce: lease.nonce, |
| 351 | description: "Alchemy Machine lifecycle", |
| 352 | }) |
| 353 | .pipe(Retry.none); |
| 354 | if (held.get(machineId) !== lease) return; |
| 355 | if (response.data?.nonce !== lease.nonce) { |
| 356 | return yield* lose(machineId, "refresh returned a different nonce"); |
| 357 | } |
| 358 | return yield* record(machineId, response, requestedAt); |
| 359 | }).pipe( |
| 360 | Effect.retry({ |
| 361 | times: 8, |
| 362 | schedule: retrySchedule<machines.CreateMachineLeaseError | MachineLeaseLost>(), |
| 363 | while: (error) => error._tag === "TooManyRequests", |
| 364 | }), |
| 365 | Effect.timeout("75 seconds"), |
| 366 | Effect.catch((error) => |
| 367 | Effect.gen(function* () { |
| 368 | if (held.get(machineId) !== lease) return; |
| 369 | uncertain.add(machineId); |
| 370 | // A missing lease during removal is not proof that the Machine is gone. |
| 371 | if (error._tag === "NotFound" && deleting.has(machineId)) { |
| 372 | const absent = yield* confirmRemoved(machineId).pipe( |
| 373 | Effect.catch((observation) => |
| 374 | held.get(machineId) === lease |
| 375 | ? lose(machineId, `removal observation uncertain: ${observation._tag}`) |
| 376 | : Effect.succeed(true), |
| 377 | ), |
| 378 | ); |
| 379 | if (absent || held.get(machineId) !== lease) return; |
| 380 | return yield* lose(machineId, "lease disappeared before removal was confirmed"); |
| 381 | } |
| 382 | return yield* lose(machineId, `refresh uncertain: ${error._tag}`); |
| 383 | }), |
| 384 | ), |
| 385 | (operation) => |
| 386 | Effect.uninterruptible( |
| 387 | Effect.gen(function* () { |
| 388 | const pending = yield* operation.pipe(Effect.interruptible, Effect.forkChild); |
| 389 | return yield* Fiber.join(pending); |
| 390 | }), |
| 391 | ), |
| 392 | ); |
| 393 | // Each target renews serially on its own cadence within the operation scope. |
| 394 | const heartbeat = (machineId: string, acquired: HeldLease) => |
| 395 | Effect.gen(function* () { |
| 396 | let lease: HeldLease | undefined = acquired; |
| 397 | while (lease && held.get(machineId) === lease) { |
| 398 | const now = yield* Clock.currentTimeMillis; |
| 399 | yield* Effect.sleep(Math.max(0, lease.renewAt - now)); |
| 400 | lease = yield* refresh(machineId, lease); |
| 401 | } |
| 402 | }).pipe(Effect.ignore); |
| 403 | yield* Effect.sleep("1 second").pipe( |
| 404 | Effect.andThen(check), |
| 405 | Effect.forever, |
| 406 | Effect.ignore, |
| 407 | Effect.forkScoped, |
| 408 | ); |
| 409 | const checkTarget = (machineId: string) => nonce(machineId).pipe(Effect.asVoid); |
| 410 | return { |
| 411 | acquire, |
| 412 | check, |
| 413 | checkTarget, |
| 414 | guard, |
| 415 | mutate, |
| 416 | remove, |
| 417 | forget, |
| 418 | nonceIfHeld, |
| 419 | }; |
| 420 | }); |
| 421 | |
| 422 | export type MachineLeases = Effect.Success<ReturnType<typeof makeMachineLeases>>; |
| 423 | |
| 424 | export const usingMachineLeases = <A, E, R>( |
| 425 | appName: string, |
| 426 | existing: MachineLeases | undefined, |
| 427 | use: (leases: MachineLeases) => Effect.Effect<A, E, R>, |
| 428 | ) => |
| 429 | Effect.gen(function* () { |
| 430 | if (existing) return yield* existing.guard(Effect.suspend(() => use(existing))); |
| 431 | return yield* Effect.scoped( |
| 432 | Effect.gen(function* () { |
| 433 | const leases = yield* makeMachineLeases(appName); |
| 434 | return yield* leases.guard(Effect.suspend(() => use(leases))); |
| 435 | }), |
| 436 | ); |
| 437 | }); |