@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 437 lines 15.7 KB
Raw
1 import {
2 BadGateway,
3 GatewayTimeout,
4 InternalServerError,
5 ServiceUnavailable,
6 TooManyRequests,
7 } from "@distilled.cloud/fly-io/Errors";
8 import * as machines from "@distilled.cloud/fly-io/machines";
9 import * as Retry from "@distilled.cloud/fly-io/Retry";
10 import * as Clock from "effect/Clock";
11 import * as Data from "effect/Data";
12 import * as Deferred from "effect/Deferred";
13 import * as Duration from "effect/Duration";
14 import * as Effect from "effect/Effect";
15 import * as Fiber from "effect/Fiber";
16 import * as FetchHttpClient from "effect/http/FetchHttpClient";
17 import * as HttpClient from "effect/http/HttpClient";
18 import * as HttpClientRequest from "effect/http/HttpClientRequest";
19 import * as Option from "effect/Option";
20 import * as Schedule from "effect/Schedule";
21
22 const TTL_SECONDS = 120;
23 const EXPIRY_MARGIN_MS = 15_000;
24
25 const retrySchedule = <E>(): Schedule.Schedule<Duration.Duration, E> =>
26 Schedule.exponential("500 millis").pipe(
27 Schedule.modifyDelay<Duration.Duration, E>(({ input, duration }) =>
28 Effect.succeed(
29 (TooManyRequests.is(input) ||
30 InternalServerError.is(input) ||
31 BadGateway.is(input) ||
32 ServiceUnavailable.is(input) ||
33 GatewayTimeout.is(input)) &&
34 input.retryAfter
35 ? Math.min(Duration.toMillis(input.retryAfter), 60_000)
36 : Math.min(Duration.toMillis(duration), 5_000),
37 ),
38 ),
39 Schedule.addDelay<Duration.Duration, E>(() =>
40 Effect.sync(() => Duration.millis(Math.random() * 50)),
41 ),
42 );
43
44 export class MachineLeaseBusy extends Data.TaggedError("Fly.MachineLeaseBusy")<{
45 appName: string;
46 machineId: string;
47 }> {}
48
49 export class MachineLeaseLost extends Data.TaggedError("Fly.MachineLeaseLost")<{
50 appName: string;
51 machineId: string;
52 reason: string;
53 }> {
54 get message() {
55 return `Machine lease authority lost for ${this.appName}/${this.machineId}: ${this.reason}`;
56 }
57 }
58
59 export class MachineMutationUncertain extends Data.TaggedError("Fly.MachineMutationUncertain")<{
60 appName: string;
61 machineId: string;
62 }> {}
63
64 interface HeldLease {
65 nonce: string;
66 deadline: number;
67 renewAt: number;
68 }
69
70 export const makeMachineLeases = Effect.fn(function* (appName: string) {
71 const scope = yield* Effect.scope;
72 const held = new Map<string, HeldLease>();
73 const deleting = new Set<string>();
74 const renewals = new Set<Fiber.Fiber<void, never>>();
75 const uncertain = new Set<string>();
76 const lost = yield* Deferred.make<never, MachineLeaseLost>();
77 let failure: MachineLeaseLost | undefined;
78 const lose = (machineId: string, reason: string) =>
79 Effect.gen(function* () {
80 failure ??= new MachineLeaseLost({ appName, machineId, reason });
81 yield* Deferred.fail(lost, failure);
82 return yield* failure;
83 });
84 const check = Effect.gen(function* () {
85 if (failure) return yield* failure;
86 const now = yield* Clock.currentTimeMillis;
87 for (const [machineId, lease] of held) {
88 if (now >= lease.deadline) return yield* lose(machineId, "lease authority expired");
89 }
90 });
91 const guard = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
92 check.pipe(Effect.andThen(Effect.raceFirst(effect, Deferred.await(lost))));
93 const record = (machineId: string, response: machines.MachineLease, requestedAt: number) =>
94 Effect.gen(function* () {
95 const lease = response.data;
96 const now = yield* Clock.currentTimeMillis;
97 const deadline =
98 Math.min((lease?.expires_at ?? 0) * 1000, requestedAt + TTL_SECONDS * 1000) -
99 EXPIRY_MARGIN_MS;
100 const captured = lease?.nonce
101 ? { nonce: lease.nonce, deadline, renewAt: requestedAt + 25_000 }
102 : undefined;
103 if (captured) held.set(machineId, captured);
104 if (!captured || !Number.isFinite(deadline) || deadline <= now) {
105 return yield* lose(machineId, "invalid or expired lease response");
106 }
107 return captured;
108 });
109 const acquire = (machineIds: readonly string[]) =>
110 Effect.gen(function* () {
111 for (const machineId of [...new Set(machineIds)].sort()) {
112 yield* check;
113 if (held.has(machineId)) continue;
114 // Save a returned nonce atomically; uncertain interrupted requests expire at Fly.
115 yield* Effect.uninterruptibleMask((restore) =>
116 Effect.gen(function* () {
117 const requestedAt = yield* Clock.currentTimeMillis;
118 const response = yield* restore(
119 machines
120 .createMachineLease({
121 app_name: appName,
122 machine_id: machineId,
123 ttl: TTL_SECONDS,
124 description: "Alchemy Machine lifecycle",
125 })
126 .pipe(
127 Retry.none,
128 Effect.retry({
129 times: 8,
130 schedule: retrySchedule<machines.CreateMachineLeaseError>(),
131 while: (error) => error._tag === "Conflict" || error._tag === "TooManyRequests",
132 }),
133 Effect.timeout("30 seconds"),
134 Effect.catchTag("Conflict", () =>
135 Effect.fail(new MachineLeaseBusy({ appName, machineId })),
136 ),
137 Effect.catchTag(["HttpClientError", "GatewayTimeout", "TimeoutError"], (error) =>
138 lose(machineId, `acquisition uncertain: ${error._tag}`),
139 ),
140 ),
141 );
142 const lease = yield* record(machineId, response, requestedAt);
143 const renewal = yield* heartbeat(machineId, lease).pipe(
144 Effect.interruptible,
145 Effect.forkIn(scope),
146 );
147 renewals.add(renewal);
148 }),
149 );
150 yield* check;
151 }
152 });
153 const nonce = (machineId: string) =>
154 Effect.gen(function* () {
155 yield* check;
156 const lease = held.get(machineId);
157 if (!lease) return yield* lose(machineId, "target is outside the acquired lease set");
158 return lease.nonce;
159 });
160 const mutate = <A, E extends { readonly _tag: string }, R>(
161 machineId: string,
162 operation: (nonce: string) => Effect.Effect<A, E, R>,
163 options: { idempotent?: boolean; timeoutMs?: number } = {},
164 ) =>
165 guard(
166 Effect.gen(function* () {
167 const value = yield* nonce(machineId);
168 return yield* operation(value).pipe(Retry.none);
169 }).pipe(
170 Effect.retry({
171 times: 8,
172 schedule: retrySchedule<E | MachineLeaseLost>(),
173 // Only explicit throttling is safe to replay for update/stop/delete.
174 while: (error) =>
175 error._tag === "TooManyRequests" ||
176 (options.idempotent === true &&
177 (error._tag === "InternalServerError" ||
178 error._tag === "BadGateway" ||
179 error._tag === "ServiceUnavailable" ||
180 error._tag === "GatewayTimeout")),
181 }),
182 Effect.timeout(options.timeoutMs ?? 120_000),
183 Effect.catch((error) =>
184 Effect.gen(function* () {
185 if (error._tag === "Conflict" || error._tag === "Forbidden") {
186 return yield* lose(machineId, `leased mutation rejected: ${error._tag}`);
187 }
188 // Transport errors retain the request, including its private nonce header.
189 if (
190 error._tag === "HttpClientError" ||
191 error._tag === "GatewayTimeout" ||
192 error._tag === "TimeoutError"
193 ) {
194 return yield* new MachineMutationUncertain({
195 appName,
196 machineId,
197 });
198 }
199 return yield* Effect.fail(error);
200 }),
201 ),
202 ),
203 );
204 const forget = (machineId: string) =>
205 Effect.sync(() => {
206 held.delete(machineId);
207 });
208 const confirmRemoved = (machineId: string) =>
209 guard(
210 machines
211 .getMachine({
212 app_name: appName,
213 machine_id: machineId,
214 })
215 .pipe(
216 Retry.none,
217 Effect.map((machine) => machine.id === machineId && machine.state === "destroyed"),
218 Effect.catchTag("NotFound", () => Effect.succeed(true)),
219 Effect.catchTag(
220 [
221 "TooManyRequests",
222 "InternalServerError",
223 "BadGateway",
224 "ServiceUnavailable",
225 "GatewayTimeout",
226 ],
227 () => Effect.succeed(false),
228 ),
229 Effect.catchReason("HttpClientError", "TransportError", () => Effect.succeed(false)),
230 Effect.repeat({
231 times: 8,
232 schedule: Schedule.spaced("2 seconds"),
233 until: (absent) => absent,
234 }),
235 Effect.timeoutOrElse({ duration: "30 seconds", orElse: () => Effect.succeed(false) }),
236 Effect.catch((error) =>
237 Effect.gen(function* () {
238 if (!held.has(machineId)) return true;
239 if (error._tag === "Forbidden") {
240 return yield* lose(machineId, "removal observation forbidden");
241 }
242 return yield* error;
243 }),
244 ),
245 // Another observer's confirmed removal wins over a late uncertain response.
246 Effect.map((absent) => absent || !held.has(machineId)),
247 Effect.tap((absent) => (absent ? forget(machineId) : Effect.void)),
248 ),
249 );
250 const remove = <A, E extends { readonly _tag: string }, R>(
251 machineId: string,
252 operation: (nonce: string) => Effect.Effect<A, E, R>,
253 ) =>
254 Effect.gen(function* () {
255 const http = yield* HttpClient.HttpClient;
256 const fetchOptions = yield* Effect.serviceOption(FetchHttpClient.RequestInit);
257 yield* Effect.sync(() => {
258 deleting.add(machineId);
259 });
260 yield* mutate(machineId, (nonce) =>
261 operation(nonce).pipe(
262 // Bun can replay DELETE on a reused socket below the SDK retry policy.
263 Effect.provideService(
264 HttpClient.HttpClient,
265 HttpClient.mapRequest(http, HttpClientRequest.setHeader("connection", "close")),
266 ),
267 Effect.provideService(FetchHttpClient.RequestInit, {
268 ...Option.getOrUndefined(fetchOptions),
269 keepalive: false,
270 redirect: "error",
271 }),
272 ),
273 ).pipe(
274 Effect.asVoid,
275 Effect.catch((error) =>
276 Effect.gen(function* () {
277 if (error._tag === "NotFound") return yield* forget(machineId);
278 if (
279 error._tag === "Fly.MachineMutationUncertain" ||
280 error._tag === "InternalServerError" ||
281 error._tag === "BadGateway" ||
282 error._tag === "ServiceUnavailable"
283 ) {
284 // Keep renewal aware of the removal until its exact target has been observed.
285 if (yield* confirmRemoved(machineId)) return;
286 }
287 return yield* Effect.fail(error);
288 }),
289 ),
290 );
291 yield* forget(machineId);
292 }).pipe(
293 Effect.ensuring(
294 Effect.sync(() => {
295 deleting.delete(machineId);
296 }),
297 ),
298 );
299 const nonceIfHeld = (machineId: string) =>
300 check.pipe(Effect.map(() => held.get(machineId)?.nonce));
301
302 yield* Effect.addFinalizer(() =>
303 Effect.gen(function* () {
304 // A late refresh can recreate a lease after a successful release.
305 yield* Fiber.interruptAll(renewals);
306 yield* Effect.forEach(
307 [...held.entries()]
308 .filter(([machineId]) => !uncertain.has(machineId))
309 .sort(([a], [b]) => a.localeCompare(b)),
310 ([machineId, lease]) =>
311 machines
312 .machinesReleaseLease({
313 app_name: appName,
314 machine_id: machineId,
315 lease_nonce: lease.nonce,
316 })
317 .pipe(
318 Retry.none,
319 Effect.timeout("5 seconds"),
320 Effect.interruptible,
321 Effect.catchTag("NotFound", () => Effect.void),
322 Effect.catch((error) =>
323 Effect.logWarning(
324 "Fly lease release was not confirmed; expiry remains the safety bound",
325 { appName, machineId, error: error._tag },
326 ),
327 ),
328 Effect.asVoid,
329 ),
330 { concurrency: 4 },
331 );
332 if (uncertain.size > 0)
333 yield* Effect.logWarning(
334 "Fly lease renewal was not confirmed; expiry remains the safety bound",
335 { appName, machineIds: [...uncertain] },
336 );
337 }),
338 );
339
340 const refresh = (machineId: string, lease: HeldLease) =>
341 Effect.gen(function* () {
342 yield* check;
343 if (held.get(machineId) !== lease) return;
344 const requestedAt = yield* Clock.currentTimeMillis;
345 const response = yield* machines
346 .createMachineLease({
347 app_name: appName,
348 machine_id: machineId,
349 ttl: TTL_SECONDS,
350 lease_nonce: lease.nonce,
351 description: "Alchemy Machine lifecycle",
352 })
353 .pipe(Retry.none);
354 if (held.get(machineId) !== lease) return;
355 if (response.data?.nonce !== lease.nonce) {
356 return yield* lose(machineId, "refresh returned a different nonce");
357 }
358 return yield* record(machineId, response, requestedAt);
359 }).pipe(
360 Effect.retry({
361 times: 8,
362 schedule: retrySchedule<machines.CreateMachineLeaseError | MachineLeaseLost>(),
363 while: (error) => error._tag === "TooManyRequests",
364 }),
365 Effect.timeout("75 seconds"),
366 Effect.catch((error) =>
367 Effect.gen(function* () {
368 if (held.get(machineId) !== lease) return;
369 uncertain.add(machineId);
370 // A missing lease during removal is not proof that the Machine is gone.
371 if (error._tag === "NotFound" && deleting.has(machineId)) {
372 const absent = yield* confirmRemoved(machineId).pipe(
373 Effect.catch((observation) =>
374 held.get(machineId) === lease
375 ? lose(machineId, `removal observation uncertain: ${observation._tag}`)
376 : Effect.succeed(true),
377 ),
378 );
379 if (absent || held.get(machineId) !== lease) return;
380 return yield* lose(machineId, "lease disappeared before removal was confirmed");
381 }
382 return yield* lose(machineId, `refresh uncertain: ${error._tag}`);
383 }),
384 ),
385 (operation) =>
386 Effect.uninterruptible(
387 Effect.gen(function* () {
388 const pending = yield* operation.pipe(Effect.interruptible, Effect.forkChild);
389 return yield* Fiber.join(pending);
390 }),
391 ),
392 );
393 // Each target renews serially on its own cadence within the operation scope.
394 const heartbeat = (machineId: string, acquired: HeldLease) =>
395 Effect.gen(function* () {
396 let lease: HeldLease | undefined = acquired;
397 while (lease && held.get(machineId) === lease) {
398 const now = yield* Clock.currentTimeMillis;
399 yield* Effect.sleep(Math.max(0, lease.renewAt - now));
400 lease = yield* refresh(machineId, lease);
401 }
402 }).pipe(Effect.ignore);
403 yield* Effect.sleep("1 second").pipe(
404 Effect.andThen(check),
405 Effect.forever,
406 Effect.ignore,
407 Effect.forkScoped,
408 );
409 const checkTarget = (machineId: string) => nonce(machineId).pipe(Effect.asVoid);
410 return {
411 acquire,
412 check,
413 checkTarget,
414 guard,
415 mutate,
416 remove,
417 forget,
418 nonceIfHeld,
419 };
420 });
421
422 export type MachineLeases = Effect.Success<ReturnType<typeof makeMachineLeases>>;
423
424 export const usingMachineLeases = <A, E, R>(
425 appName: string,
426 existing: MachineLeases | undefined,
427 use: (leases: MachineLeases) => Effect.Effect<A, E, R>,
428 ) =>
429 Effect.gen(function* () {
430 if (existing) return yield* existing.guard(Effect.suspend(() => use(existing)));
431 return yield* Effect.scoped(
432 Effect.gen(function* () {
433 const leases = yield* makeMachineLeases(appName);
434 return yield* leases.guard(Effect.suspend(() => use(leases)));
435 }),
436 );
437 });