The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import type * as lambda from "aws-lambda"; |
| 2 | import * as Clock from "effect/Clock"; |
| 3 | import * as Context from "effect/Context"; |
| 4 | import * as Data from "effect/Data"; |
| 5 | import * as Duration from "effect/Duration"; |
| 6 | import * as Effect from "effect/Effect"; |
| 7 | import * as Exit from "effect/Exit"; |
| 8 | import * as Fiber from "effect/Fiber"; |
| 9 | import { Flusher } from "effect/observability/OtlpExporter"; |
| 10 | import * as Option from "effect/Option"; |
| 11 | |
| 12 | /** |
| 13 | * The AWS Lambda invocation context — the `context` argument Lambda passes |
| 14 | * to the handler. Provided to every invocation's effect by the Lambda |
| 15 | * dispatcher (see `Function.ts`). |
| 16 | */ |
| 17 | export class HandlerContext extends Context.Service<HandlerContext, lambda.Context>()( |
| 18 | "AWS.Lambda.HandlerContext", |
| 19 | ) {} |
| 20 | |
| 21 | /** |
| 22 | * Environment variable the deploy side writes and the runtime reads per |
| 23 | * invocation: the milliseconds between the Function's `timeout` and the |
| 24 | * timeout Lambda is configured with. The runtime interrupts the handler that |
| 25 | * long before Lambda would kill it. Unset or `0` → the handler is never |
| 26 | * interrupted. |
| 27 | */ |
| 28 | export const TIMEOUT_GRACE_ENV = "ALCHEMY_LAMBDA_TIMEOUT_GRACE_MS"; |
| 29 | |
| 30 | /** |
| 31 | * Default {@link FunctionCommonProps.timeoutMargin}: the time Lambda keeps |
| 32 | * the invocation alive after the handler is interrupted, for finalizers and |
| 33 | * the telemetry flush. |
| 34 | */ |
| 35 | export const DEFAULT_TIMEOUT_MARGIN_MS = 1000; |
| 36 | |
| 37 | /** |
| 38 | * How long before Lambda's hard timeout the backstop flush runs when |
| 39 | * finalizers are still running (or the handler is not interrupted at all). |
| 40 | * Wide enough for one OTLP export round-trip to a remote collector. |
| 41 | */ |
| 42 | export const BACKSTOP_FLUSH_MS = 500; |
| 43 | |
| 44 | /** |
| 45 | * Span attribute set on a root span the backstop flush ended early: the |
| 46 | * invocation was still running moments before Lambda's hard timeout, so the |
| 47 | * span was exported before Lambda could kill it. |
| 48 | */ |
| 49 | export const TIMEOUT_IMMINENT_ATTRIBUTE = "aws.lambda.timeout.imminent"; |
| 50 | |
| 51 | /** |
| 52 | * The invocation ran past the Function's `timeout`. The runtime interrupts |
| 53 | * the handler — so `acquireRelease`, `ensuring` and scope finalizers run, and |
| 54 | * every open span ends — and fails the invocation with this error, which |
| 55 | * becomes the status of the root span. The invocation is then held until |
| 56 | * Lambda's own timeout ends it, so Lambda reports a real timeout. |
| 57 | * |
| 58 | * Also the status of a root span the backstop flush ends just before Lambda's |
| 59 | * hard timeout. |
| 60 | */ |
| 61 | export class InvocationTimeoutError extends Data.TaggedError("AWS.Lambda.InvocationTimeoutError")<{ |
| 62 | readonly functionName: string; |
| 63 | readonly requestId: string; |
| 64 | /** Milliseconds the handler had been running when the deadline fired. */ |
| 65 | readonly budgetMs: number; |
| 66 | }> { |
| 67 | override get message() { |
| 68 | return `Lambda invocation ${this.requestId} of ${this.functionName} timed out after ${this.budgetMs}ms`; |
| 69 | } |
| 70 | } |
| 71 | |
| 72 | /** |
| 73 | * Set by the deadline guard when it interrupts an invocation at the |
| 74 | * Function's `timeout`. The dispatcher provides a fresh one per invocation |
| 75 | * and, once finalizers have run and telemetry has flushed, holds a fired |
| 76 | * invocation open until Lambda's own timeout ends it (see |
| 77 | * {@link withInvocationDeadline}). |
| 78 | */ |
| 79 | export class InvocationDeadlineState extends Context.Service< |
| 80 | InvocationDeadlineState, |
| 81 | { fired: boolean } |
| 82 | >()("AWS.Lambda.InvocationDeadlineState") {} |
| 83 | |
| 84 | interface Deadline { |
| 85 | /** Tell the enclosing guard an inner guard now owns the deadline. */ |
| 86 | readonly claim: () => void; |
| 87 | } |
| 88 | |
| 89 | /** |
| 90 | * Internal handshake between nested deadline guards. The dispatcher guards |
| 91 | * every listener; the HTTP listener guards again *inside* its `http.server` |
| 92 | * span so the timeout ends that span; a durable listener guards again to opt |
| 93 | * out of interruption. The inner guard claims the deadline and the outer one |
| 94 | * stands down, so the deadline is enforced exactly once, by the innermost |
| 95 | * guard. |
| 96 | */ |
| 97 | class InvocationDeadline extends Context.Service<InvocationDeadline, Deadline>()( |
| 98 | "AWS.Lambda.InvocationDeadline", |
| 99 | ) {} |
| 100 | |
| 101 | /** |
| 102 | * Parse the grace from its environment variable. Unset, unparseable or |
| 103 | * negative → `0` (never interrupt). |
| 104 | */ |
| 105 | export const readTimeoutGrace = (raw: string | undefined): number => { |
| 106 | if (raw === undefined || raw === "") return 0; |
| 107 | const ms = Number(raw); |
| 108 | return Number.isFinite(ms) && ms >= 0 ? Math.floor(ms) : 0; |
| 109 | }; |
| 110 | |
| 111 | /** |
| 112 | * Normalize a `Duration` prop (`timeout`, `timeoutMargin`) to whole |
| 113 | * milliseconds. Handles the state-JSON `Duration` shape |
| 114 | * (`{_id:"Duration",_tag:"Millis"|"Nanos"|"Infinity",...}`), which is not a |
| 115 | * valid `Duration.Input`. Infinite → `undefined` (default). |
| 116 | */ |
| 117 | export const toDurationMillis = (duration: Duration.Duration | undefined): number | undefined => { |
| 118 | if (duration === undefined) return undefined; |
| 119 | const json = duration as { |
| 120 | _id?: unknown; |
| 121 | _tag?: "Millis" | "Nanos" | "Infinity" | "NegativeInfinity"; |
| 122 | millis?: number; |
| 123 | nanos?: string; |
| 124 | }; |
| 125 | const input: Duration.Input = |
| 126 | json._id === "Duration" |
| 127 | ? json._tag === "Millis" |
| 128 | ? json.millis! |
| 129 | : json._tag === "Nanos" |
| 130 | ? BigInt(json.nanos!) |
| 131 | : "Infinity" |
| 132 | : duration; |
| 133 | const millis = Duration.toMillis(input); |
| 134 | return Number.isFinite(millis) ? Math.max(0, Math.ceil(millis)) : undefined; |
| 135 | }; |
| 136 | |
| 137 | /** Lambda's maximum function timeout, in seconds. */ |
| 138 | export const MAX_LAMBDA_TIMEOUT_SECONDS = 900; |
| 139 | |
| 140 | /** |
| 141 | * The function's `timeout` plus `timeoutMargin` does not fit in Lambda's |
| 142 | * 900-second maximum. |
| 143 | */ |
| 144 | export class FunctionTimeoutTooLong extends Data.TaggedError("AWS.Lambda.FunctionTimeoutTooLong")<{ |
| 145 | readonly id: string; |
| 146 | readonly timeoutMs: number; |
| 147 | readonly marginMs: number; |
| 148 | }> { |
| 149 | override get message() { |
| 150 | return `Function(${this.id}): timeout (${this.timeoutMs}ms) plus timeoutMargin (${this.marginMs}ms) exceeds Lambda's ${MAX_LAMBDA_TIMEOUT_SECONDS}s maximum. Lower the timeout to ${MAX_LAMBDA_TIMEOUT_SECONDS * 1000 - this.marginMs}ms or less, or set timeoutMargin to Duration.zero to let Lambda kill the invocation at the timeout without interrupting it.`; |
| 151 | } |
| 152 | } |
| 153 | |
| 154 | /** |
| 155 | * The Lambda configuration a Function's `timeout` and `timeoutMargin` |
| 156 | * produce. |
| 157 | * |
| 158 | * With interruption on (`interrupt: true`, a non-zero margin), Lambda's |
| 159 | * timeout is the Function's `timeout` plus the margin, rounded up to whole |
| 160 | * seconds, and `graceMs` is the time between the two: the runtime |
| 161 | * interrupts the handler at exactly `timeout` and Lambda kills it `graceMs` |
| 162 | * later. A handler that would have finished inside `timeout` is never cut |
| 163 | * short. |
| 164 | * |
| 165 | * Without it (a zero margin, or a handler alchemy's runtime does not run), |
| 166 | * Lambda's timeout is `timeout` rounded up to whole seconds and `graceMs` is |
| 167 | * `0`. |
| 168 | */ |
| 169 | export const lambdaTimeoutOf = (props: { |
| 170 | id: string; |
| 171 | timeout: Duration.Duration | undefined; |
| 172 | timeoutMargin: Duration.Duration | undefined; |
| 173 | interrupt: boolean; |
| 174 | }): Effect.Effect<{ timeoutSeconds: number; graceMs: number }, FunctionTimeoutTooLong> => { |
| 175 | const timeoutMs = toDurationMillis(props.timeout) ?? 3000; |
| 176 | const marginMs = props.interrupt |
| 177 | ? (toDurationMillis(props.timeoutMargin) ?? DEFAULT_TIMEOUT_MARGIN_MS) |
| 178 | : 0; |
| 179 | const timeoutSeconds = Math.max(1, Math.ceil((timeoutMs + marginMs) / 1000)); |
| 180 | if (timeoutSeconds > MAX_LAMBDA_TIMEOUT_SECONDS && marginMs > 0) { |
| 181 | return Effect.fail(new FunctionTimeoutTooLong({ id: props.id, timeoutMs, marginMs })); |
| 182 | } |
| 183 | return Effect.succeed({ |
| 184 | timeoutSeconds, |
| 185 | graceMs: marginMs > 0 ? timeoutSeconds * 1000 - timeoutMs : 0, |
| 186 | }); |
| 187 | }; |
| 188 | |
| 189 | /** |
| 190 | * Enforce the Function's `timeout` on an invocation, the way `runMain` |
| 191 | * treats SIGTERM: interrupt the handler so its finalizers run and its spans |
| 192 | * end, then let Lambda's extra margin carry the cleanup and the telemetry |
| 193 | * flush. |
| 194 | * |
| 195 | * The deploy side configures Lambda's timeout as the Function's `timeout` |
| 196 | * plus `timeoutMargin` and writes the difference to |
| 197 | * `ALCHEMY_LAMBDA_TIMEOUT_GRACE_MS`. The deadline is known up front |
| 198 | * (`context.getRemainingTimeInMillis()`), and a sandbox only ever handles |
| 199 | * one invocation at a time, so the timers are race-free: |
| 200 | * |
| 201 | * 1. At `remaining - grace` — the Function's `timeout` — the handler is |
| 202 | * interrupted. `acquireRelease`, `ensuring` and scope finalizers run; |
| 203 | * every open span ends with the interruption, so the trace shows the |
| 204 | * call that hung. The invocation fails with {@link InvocationTimeoutError} |
| 205 | * and {@link InvocationDeadlineState} is marked fired. A handler that |
| 206 | * finishes before its `timeout` is never touched. |
| 207 | * 2. The dispatcher closes the request scope (request finalizers, the |
| 208 | * telemetry flush) and then holds the invocation open instead of |
| 209 | * returning. Lambda's own timeout ends it: Lambda logs `Task timed out`, |
| 210 | * counts a timeout, retries as it would for any timeout, and resets the |
| 211 | * sandbox, so work that ignored the interruption (a promise without an |
| 212 | * `AbortSignal`, a connection left mid-query) can never resume in the |
| 213 | * next invocation. The cost is the unused rest of the margin, billed only |
| 214 | * on invocations that time out. |
| 215 | * 3. At {@link BACKSTOP_FLUSH_MS} before Lambda's hard timeout, if the |
| 216 | * invocation is *still* running (a slow finalizer, an uninterruptible |
| 217 | * region, or interruption is off), the current root span is ended with |
| 218 | * {@link InvocationTimeoutError} and marked |
| 219 | * {@link TIMEOUT_IMMINENT_ATTRIBUTE}, a warning is logged, and every |
| 220 | * exporter is drained through the OTLP {@link Flusher} — so the trace is |
| 221 | * exported even if Lambda kills the process. This step never changes the |
| 222 | * invocation's outcome. |
| 223 | * |
| 224 | * `interrupt: false` keeps only step 3. Durable functions use it: their |
| 225 | * invocations are replayed from checkpoints, and interrupting one would |
| 226 | * leave the durable SDK's promise running. |
| 227 | * |
| 228 | * Without a `HandlerContext` (unit tests, non-Lambda hosts) the effect runs |
| 229 | * unguarded. |
| 230 | * |
| 231 | * Guards nest: an inner guard (the HTTP listener's, placed inside the |
| 232 | * `http.server` span so the timeout ends it; the durable listener's, which |
| 233 | * turns interruption off) takes the deadline over from the outer one (the |
| 234 | * dispatcher's), so the deadline is enforced once, by the innermost guard. |
| 235 | */ |
| 236 | export const withInvocationDeadline: { |
| 237 | (options?: { |
| 238 | readonly interrupt?: boolean; |
| 239 | }): <A, E, R>(self: Effect.Effect<A, E, R>) => Effect.Effect<A, E | InvocationTimeoutError, R>; |
| 240 | } = |
| 241 | (options) => |
| 242 | <A, E, R>(self: Effect.Effect<A, E, R>) => |
| 243 | Effect.gen(function* () { |
| 244 | const context = Option.getOrUndefined(yield* Effect.serviceOption(HandlerContext)); |
| 245 | if (context === undefined || typeof context.getRemainingTimeInMillis !== "function") { |
| 246 | return yield* self; |
| 247 | } |
| 248 | const remainingMs = context.getRemainingTimeInMillis(); |
| 249 | const graceMs = |
| 250 | options?.interrupt === false ? 0 : readTimeoutGrace(process.env[TIMEOUT_GRACE_ENV]); |
| 251 | const budgetMs = remainingMs - graceMs; |
| 252 | // Leave room for the interrupt to land before the backstop; with no |
| 253 | // grace the backstop carves its margin out of the budget, as before. |
| 254 | const backstopMs = |
| 255 | graceMs > 0 ? Math.min(BACKSTOP_FLUSH_MS, Math.floor(graceMs / 2)) : BACKSTOP_FLUSH_MS; |
| 256 | const backstopAtMs = remainingMs - backstopMs; |
| 257 | |
| 258 | // Take the deadline over from any enclosing guard. |
| 259 | const outer = yield* Effect.serviceOption(InvocationDeadline); |
| 260 | if (Option.isSome(outer)) { |
| 261 | outer.value.claim(); |
| 262 | } |
| 263 | let claimed = false; |
| 264 | const deadline: Deadline = { |
| 265 | claim: () => { |
| 266 | claimed = true; |
| 267 | }, |
| 268 | }; |
| 269 | |
| 270 | const timeoutError = new InvocationTimeoutError({ |
| 271 | functionName: context.functionName, |
| 272 | requestId: context.awsRequestId, |
| 273 | budgetMs, |
| 274 | }); |
| 275 | |
| 276 | const backstopFlush = Effect.gen(function* () { |
| 277 | // The root span is still open (that is why we are here); end it so |
| 278 | // it exports with the timeout as its status. Only `Span`s can be |
| 279 | // ended — an external parent (a propagated `traceparent` with no |
| 280 | // local span) is skipped. |
| 281 | const span = yield* Effect.option(Effect.currentSpan); |
| 282 | if (Option.isSome(span) && span.value.status._tag === "Started") { |
| 283 | const root = span.value; |
| 284 | root.attribute(TIMEOUT_IMMINENT_ATTRIBUTE, true); |
| 285 | root.end(yield* Clock.currentTimeNanos, Exit.fail(timeoutError)); |
| 286 | // This export is the span's final record. Tracers export on every |
| 287 | // `end`, so if the invocation finishes before Lambda kills it the |
| 288 | // tracer's own `end` would ship the span a second time; drop it. |
| 289 | root.end = () => {}; |
| 290 | } |
| 291 | yield* Effect.logWarning( |
| 292 | `Lambda invocation ${context.awsRequestId} of ${context.functionName} is still running ${backstopMs}ms before Lambda's timeout; telemetry was flushed early in case Lambda kills the invocation`, |
| 293 | ).pipe(Effect.annotateLogs({ requestId: context.awsRequestId })); |
| 294 | const flusher = yield* Effect.serviceOption(Flusher); |
| 295 | if (Option.isSome(flusher)) { |
| 296 | yield* flusher.value.flush; |
| 297 | } |
| 298 | }); |
| 299 | |
| 300 | // The backstop inherits this fiber's context — the invocation's |
| 301 | // logger, exporters and (on the HTTP path) the root span — and is |
| 302 | // interrupted when the invocation settles. It is forked outside the |
| 303 | // timeout below, so it keeps running while a slow finalizer holds up |
| 304 | // the interrupted handler. The flush itself is uninterruptible so an |
| 305 | // invocation finishing mid-flush waits for the export rather than |
| 306 | // abandoning an in-flight batch. |
| 307 | const backstop = |
| 308 | backstopAtMs > 0 |
| 309 | ? yield* Effect.sleep(Duration.millis(backstopAtMs)).pipe( |
| 310 | Effect.andThen( |
| 311 | Effect.suspend(() => |
| 312 | claimed ? Effect.void : Effect.uninterruptible(backstopFlush), |
| 313 | ), |
| 314 | ), |
| 315 | Effect.forkChild, |
| 316 | ) |
| 317 | : undefined; |
| 318 | |
| 319 | const guarded = self.pipe(Effect.provideService(InvocationDeadline, deadline)); |
| 320 | // Interrupt at the Function's `timeout`. The timer loses the race to |
| 321 | // the handler once an inner guard claims the deadline — that guard |
| 322 | // enforces it closer to the work, at the same instant — so the |
| 323 | // deadline fails the invocation once, from the innermost guard. The |
| 324 | // race interrupts the handler and waits for its finalizers. |
| 325 | const timer = Effect.sleep(Duration.millis(budgetMs)).pipe( |
| 326 | Effect.andThen( |
| 327 | Effect.suspend(() => |
| 328 | claimed |
| 329 | ? Effect.never |
| 330 | : Effect.logWarning(timeoutError.message).pipe( |
| 331 | Effect.annotateLogs({ |
| 332 | requestId: timeoutError.requestId, |
| 333 | budgetMs: timeoutError.budgetMs, |
| 334 | }), |
| 335 | Effect.andThen(Effect.serviceOption(InvocationDeadlineState)), |
| 336 | Effect.tap((state) => |
| 337 | Effect.sync(() => { |
| 338 | if (Option.isSome(state)) state.value.fired = true; |
| 339 | }), |
| 340 | ), |
| 341 | Effect.andThen(Effect.fail(timeoutError)), |
| 342 | ), |
| 343 | ), |
| 344 | ), |
| 345 | ); |
| 346 | return yield* (graceMs > 0 && budgetMs > 0 ? Effect.raceFirst(guarded, timer) : guarded).pipe( |
| 347 | Effect.ensuring(backstop ? Fiber.interrupt(backstop) : Effect.void), |
| 348 | ); |
| 349 | }); |