The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import * as AWS from "@alchemy.run/aws"; |
| 2 | import * as Cloudflare from "@alchemy.run/cloudflare"; |
| 3 | import { describe, expect } from "@alchemy.run/test"; |
| 4 | import * as Test from "@alchemy.run/test/Alchemy"; |
| 5 | import * as logs from "@distilled.cloud/aws/cloudwatch-logs"; |
| 6 | import * as Clock from "effect/Clock"; |
| 7 | import * as ConfigProvider from "effect/ConfigProvider"; |
| 8 | import * as Effect from "effect/Effect"; |
| 9 | import * as HttpClient from "effect/http/HttpClient"; |
| 10 | import * as Layer from "effect/Layer"; |
| 11 | import * as Schedule from "effect/Schedule"; |
| 12 | import * as pathe from "pathe"; |
| 13 | import { expectUrlContains } from "../../Cloudflare/Utils/Http.ts"; |
| 14 | import type { OtelSink } from "./fixtures/otel-collector-worker.ts"; |
| 15 | import { OtelTestFunction, OtelTestFunctionLive } from "./fixtures/otel-handler.ts"; |
| 16 | |
| 17 | const { test } = Test.make({ providers: Layer.mergeAll(AWS.providers(), Cloudflare.providers()) }); |
| 18 | |
| 19 | const collectorMain = pathe.resolve(import.meta.dirname, "fixtures/otel-collector-worker.ts"); |
| 20 | |
| 21 | // The OTLP sink the Lambda exports to. A workers.dev URL is fine here — |
| 22 | // the 1042 worker-to-worker restriction doesn't apply to requests coming |
| 23 | // from Lambda. Declared identically in both deploy steps so the second |
| 24 | // deploy keeps it. |
| 25 | const collectorWorker = () => |
| 26 | Cloudflare.Worker("OtelCollector", { |
| 27 | main: collectorMain, |
| 28 | env: { SINK: Cloudflare.DurableObject<OtelSink>("OtelSink") }, |
| 29 | compatibility: { date: "2024-09-23" }, |
| 30 | }); |
| 31 | |
| 32 | interface CollectedSpan { |
| 33 | name: string; |
| 34 | kind?: number; |
| 35 | status?: { code?: number; message?: string }; |
| 36 | attributes?: { key: string; value: Record<string, unknown> }[]; |
| 37 | } |
| 38 | |
| 39 | // Every span across all the OTLP traces pushes the collector recorded. |
| 40 | const collectedSpans = (collected: unknown): CollectedSpan[] => |
| 41 | (collected as { items: { signal: string; payload: any }[] }).items |
| 42 | .filter((item) => item.signal === "traces") |
| 43 | .flatMap((item) => item.payload.resourceSpans ?? []) |
| 44 | .flatMap((resource: any) => resource.scopeSpans ?? []) |
| 45 | .flatMap((scope: any) => (scope.spans ?? []) as CollectedSpan[]); |
| 46 | |
| 47 | // Name, path and status of each span, printed when an assertion fails. |
| 48 | const rootSpanSummary = (collected: unknown) => |
| 49 | collectedSpans(collected).map((span) => ({ |
| 50 | name: span.name, |
| 51 | path: span.attributes?.find((attribute) => attribute.key === "url.path")?.value, |
| 52 | status: span.status?.code, |
| 53 | })); |
| 54 | |
| 55 | // OTLP SPAN_KIND_SERVER: the HttpMiddleware tracer's root span per request. |
| 56 | const SERVER_SPAN_KIND = 2; |
| 57 | |
| 58 | // The HTTP server root spans for one path. |
| 59 | const rootSpansFor = (collected: unknown, path: string): CollectedSpan[] => |
| 60 | collectedSpans(collected).filter( |
| 61 | (span) => |
| 62 | span.kind === SERVER_SPAN_KIND && |
| 63 | span.attributes?.some( |
| 64 | (attribute) => attribute.key === "url.path" && attribute.value.stringValue === path, |
| 65 | ), |
| 66 | ); |
| 67 | |
| 68 | describe( |
| 69 | "AWS.Lambda Telemetry", |
| 70 | { |
| 71 | tags: [ |
| 72 | "provider:aws", |
| 73 | "provider:aws:lambda", |
| 74 | "provider:cloudflare", |
| 75 | "provider:cloudflare:worker", |
| 76 | "live", |
| 77 | ], |
| 78 | }, |
| 79 | () => { |
| 80 | test.provider( |
| 81 | "Lambda exports OTLP telemetry per invocation", |
| 82 | (stack) => |
| 83 | Effect.gen(function* () { |
| 84 | yield* stack.destroy(); |
| 85 | |
| 86 | // Deploy the collector first: the Lambda's OTLP endpoint is |
| 87 | // resolved from the deployer's environment at deploy time. |
| 88 | const collector = yield* stack.deploy( |
| 89 | Effect.gen(function* () { |
| 90 | return yield* collectorWorker(); |
| 91 | }), |
| 92 | ); |
| 93 | const collected = `${collector.url}/collected`; |
| 94 | yield* expectUrlContains(collected, "otel-collector-ok", { timeout: "180 seconds" }); |
| 95 | |
| 96 | // The fixture's telemetry binding layer reads `COLLECTOR_URL` via |
| 97 | // `Config` at Init. The test harness snapshots its ConfigProvider |
| 98 | // before the test body runs, so layer the just-learned collector |
| 99 | // URL on top of the current provider for this deploy only. |
| 100 | const currentConfig = yield* ConfigProvider.ConfigProvider; |
| 101 | const { fn } = yield* stack |
| 102 | .deploy( |
| 103 | Effect.gen(function* () { |
| 104 | yield* collectorWorker(); |
| 105 | const fn = yield* OtelTestFunction.pipe(Effect.provide(OtelTestFunctionLive)); |
| 106 | return { fn }; |
| 107 | }), |
| 108 | ) |
| 109 | .pipe( |
| 110 | Effect.provideService( |
| 111 | ConfigProvider.ConfigProvider, |
| 112 | ConfigProvider.orElse( |
| 113 | ConfigProvider.fromUnknown({ COLLECTOR_URL: collector.url }), |
| 114 | currentConfig, |
| 115 | ), |
| 116 | ), |
| 117 | ); |
| 118 | |
| 119 | // Wait for the function URL to serve AND for the collector to be |
| 120 | // reachable from the Lambda's region — workers.dev propagates |
| 121 | // per-PoP, so the Lambda can see placeholder 404s long after the |
| 122 | // test machine gets 200s. |
| 123 | const fnUrl = (fn.functionUrl as string).replace(/\/$/, ""); |
| 124 | yield* expectUrlContains(`${fnUrl}/probe`, '"status":200', { |
| 125 | timeout: "240 seconds", |
| 126 | label: "collector reachable from Lambda", |
| 127 | }); |
| 128 | |
| 129 | // Drive one traced invocation. |
| 130 | yield* expectUrlContains(`${fnUrl}/work`, "lambda-did-work", { timeout: "120 seconds" }); |
| 131 | |
| 132 | // The invocation scope's flush ships everything the invocation |
| 133 | // produced before the response is returned. |
| 134 | yield* expectUrlContains(collected, "otel-lambda-test", { |
| 135 | timeout: "120 seconds", |
| 136 | label: "lambda service.name", |
| 137 | }); |
| 138 | // HTTP server root span from the HttpMiddleware tracer, named |
| 139 | // after the request method. |
| 140 | yield* expectUrlContains(collected, '"name":"GET"'); |
| 141 | // Child span from Effect.fn instrumentation. |
| 142 | yield* expectUrlContains(collected, "lambda.child-span"); |
| 143 | // Log record shipped by the OTLP logger. |
| 144 | yield* expectUrlContains(collected, "lambda-work-log"); |
| 145 | |
| 146 | // An invocation that runs past its 3 s timeout is interrupted: |
| 147 | // its finalizer runs, the span of the call that hung ends, and the |
| 148 | // root span ends with the timeout — all exported. The invocation |
| 149 | // is then held until Lambda's own 5 s timeout ends it, so the |
| 150 | // caller sees Lambda's timeout error, not a response from the |
| 151 | // handler. |
| 152 | const client = yield* HttpClient.HttpClient; |
| 153 | const slowStart = yield* Clock.currentTimeMillis; |
| 154 | const slow = yield* client.get(`${fnUrl}/slow`); |
| 155 | const slowElapsed = (yield* Clock.currentTimeMillis) - slowStart; |
| 156 | expect(slow.status).not.toBe(200); |
| 157 | expect(slowElapsed).toBeGreaterThanOrEqual(4500); |
| 158 | |
| 159 | // Lambda counts it as a real timeout and shuts the sandbox down |
| 160 | // (our shutdown extension logs the SIGTERM inside the invocation), |
| 161 | // so work that ignored the interruption cannot resume later. |
| 162 | const logGroupName = `/aws/lambda/${fn.functionName}`; |
| 163 | const slowRequestLines = yield* logs.filterLogEvents({ logGroupName }).pipe( |
| 164 | Effect.map((result) => (result.events ?? []).map((event) => event.message ?? "")), |
| 165 | Effect.map((lines) => { |
| 166 | const requestId = lines |
| 167 | .map((line) => /Lambda invocation (\S+) of .* timed out after/.exec(line)?.[1]) |
| 168 | .find((id) => id !== undefined); |
| 169 | return requestId === undefined |
| 170 | ? [] |
| 171 | : lines.filter((line) => line.includes(requestId)); |
| 172 | }), |
| 173 | Effect.repeat({ |
| 174 | schedule: Schedule.spaced("2 seconds"), |
| 175 | until: (lines) => lines.some((line) => line.startsWith("REPORT")), |
| 176 | times: 30, |
| 177 | }), |
| 178 | ); |
| 179 | expect(slowRequestLines.find((line) => line.startsWith("REPORT"))).toContain( |
| 180 | "Status: timeout", |
| 181 | ); |
| 182 | expect(slowRequestLines.some((line) => line.includes("SIGTERM"))).toBe(true); |
| 183 | yield* expectUrlContains(collected, "lambda-slow-finalizer", { |
| 184 | timeout: "120 seconds", |
| 185 | label: "interrupted invocation's finalizer log", |
| 186 | }); |
| 187 | yield* expectUrlContains(collected, "lambda-slow-log"); |
| 188 | const slowBody = yield* client |
| 189 | .get(collected) |
| 190 | .pipe(Effect.flatMap((response) => response.json)); |
| 191 | const slowRoots = rootSpansFor(slowBody, "/slow"); |
| 192 | expect({ slowRoots: slowRoots.length, roots: rootSpanSummary(slowBody) }).toMatchObject({ |
| 193 | slowRoots: 1, |
| 194 | }); |
| 195 | expect(slowRoots[0]?.status).toMatchObject({ |
| 196 | code: 2, |
| 197 | message: expect.stringContaining("timed out"), |
| 198 | }); |
| 199 | // The span of the call that hung, ended by the interruption. |
| 200 | const hung = collectedSpans(slowBody).filter((span) => span.name === "lambda.slow-span"); |
| 201 | expect(hung).toHaveLength(1); |
| 202 | expect(hung[0]?.attributes).toContainEqual({ |
| 203 | key: "effect.fiber.interrupted", |
| 204 | value: { boolValue: true }, |
| 205 | }); |
| 206 | |
| 207 | // An invocation that finishes just inside its timeout is never |
| 208 | // interrupted, and its root span is a plain success. |
| 209 | const late = yield* client.get(`${fnUrl}/late`); |
| 210 | expect(late.status).toBe(200); |
| 211 | expect(yield* late.text).toContain("lambda-late-done"); |
| 212 | const lateBody = yield* client |
| 213 | .get(collected) |
| 214 | .pipe(Effect.flatMap((response) => response.json)); |
| 215 | const lateSpans = rootSpansFor(lateBody, "/late"); |
| 216 | expect({ lateSpans: lateSpans.length, roots: rootSpanSummary(lateBody) }).toMatchObject({ |
| 217 | lateSpans: 1, |
| 218 | }); |
| 219 | expect(lateSpans[0]?.status?.code).not.toBe(2); |
| 220 | |
| 221 | // An invocation whose finalizer hangs past Lambda's own timeout: |
| 222 | // Lambda kills it, but 500 ms before that the backstop flush ended |
| 223 | // the root span and drained the exporters. |
| 224 | const stuck = yield* client.get(`${fnUrl}/stuck`); |
| 225 | expect(stuck.status).not.toBe(200); |
| 226 | const stuckSpans = yield* client.get(collected).pipe( |
| 227 | Effect.flatMap((response) => response.json), |
| 228 | Effect.map((body) => rootSpansFor(body, "/stuck")), |
| 229 | Effect.repeat({ |
| 230 | schedule: Schedule.spaced("2 seconds"), |
| 231 | until: (spans) => spans.length > 0, |
| 232 | times: 30, |
| 233 | }), |
| 234 | ); |
| 235 | expect(stuckSpans).toHaveLength(1); |
| 236 | expect(stuckSpans[0]?.status?.code).toBe(2); |
| 237 | expect(stuckSpans[0]?.attributes).toContainEqual({ |
| 238 | key: "aws.lambda.timeout.imminent", |
| 239 | value: { boolValue: true }, |
| 240 | }); |
| 241 | }), |
| 242 | { timeout: 600_000 }, |
| 243 | ); |
| 244 | }, |
| 245 | ); |