@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 245 lines 10.6 KB
Raw
1 import * as AWS from "@alchemy.run/aws";
2 import * as Cloudflare from "@alchemy.run/cloudflare";
3 import { describe, expect } from "@alchemy.run/test";
4 import * as Test from "@alchemy.run/test/Alchemy";
5 import * as logs from "@distilled.cloud/aws/cloudwatch-logs";
6 import * as Clock from "effect/Clock";
7 import * as ConfigProvider from "effect/ConfigProvider";
8 import * as Effect from "effect/Effect";
9 import * as HttpClient from "effect/http/HttpClient";
10 import * as Layer from "effect/Layer";
11 import * as Schedule from "effect/Schedule";
12 import * as pathe from "pathe";
13 import { expectUrlContains } from "../../Cloudflare/Utils/Http.ts";
14 import type { OtelSink } from "./fixtures/otel-collector-worker.ts";
15 import { OtelTestFunction, OtelTestFunctionLive } from "./fixtures/otel-handler.ts";
16
17 const { test } = Test.make({ providers: Layer.mergeAll(AWS.providers(), Cloudflare.providers()) });
18
19 const collectorMain = pathe.resolve(import.meta.dirname, "fixtures/otel-collector-worker.ts");
20
21 // The OTLP sink the Lambda exports to. A workers.dev URL is fine here —
22 // the 1042 worker-to-worker restriction doesn't apply to requests coming
23 // from Lambda. Declared identically in both deploy steps so the second
24 // deploy keeps it.
25 const collectorWorker = () =>
26 Cloudflare.Worker("OtelCollector", {
27 main: collectorMain,
28 env: { SINK: Cloudflare.DurableObject<OtelSink>("OtelSink") },
29 compatibility: { date: "2024-09-23" },
30 });
31
32 interface CollectedSpan {
33 name: string;
34 kind?: number;
35 status?: { code?: number; message?: string };
36 attributes?: { key: string; value: Record<string, unknown> }[];
37 }
38
39 // Every span across all the OTLP traces pushes the collector recorded.
40 const collectedSpans = (collected: unknown): CollectedSpan[] =>
41 (collected as { items: { signal: string; payload: any }[] }).items
42 .filter((item) => item.signal === "traces")
43 .flatMap((item) => item.payload.resourceSpans ?? [])
44 .flatMap((resource: any) => resource.scopeSpans ?? [])
45 .flatMap((scope: any) => (scope.spans ?? []) as CollectedSpan[]);
46
47 // Name, path and status of each span, printed when an assertion fails.
48 const rootSpanSummary = (collected: unknown) =>
49 collectedSpans(collected).map((span) => ({
50 name: span.name,
51 path: span.attributes?.find((attribute) => attribute.key === "url.path")?.value,
52 status: span.status?.code,
53 }));
54
55 // OTLP SPAN_KIND_SERVER: the HttpMiddleware tracer's root span per request.
56 const SERVER_SPAN_KIND = 2;
57
58 // The HTTP server root spans for one path.
59 const rootSpansFor = (collected: unknown, path: string): CollectedSpan[] =>
60 collectedSpans(collected).filter(
61 (span) =>
62 span.kind === SERVER_SPAN_KIND &&
63 span.attributes?.some(
64 (attribute) => attribute.key === "url.path" && attribute.value.stringValue === path,
65 ),
66 );
67
68 describe(
69 "AWS.Lambda Telemetry",
70 {
71 tags: [
72 "provider:aws",
73 "provider:aws:lambda",
74 "provider:cloudflare",
75 "provider:cloudflare:worker",
76 "live",
77 ],
78 },
79 () => {
80 test.provider(
81 "Lambda exports OTLP telemetry per invocation",
82 (stack) =>
83 Effect.gen(function* () {
84 yield* stack.destroy();
85
86 // Deploy the collector first: the Lambda's OTLP endpoint is
87 // resolved from the deployer's environment at deploy time.
88 const collector = yield* stack.deploy(
89 Effect.gen(function* () {
90 return yield* collectorWorker();
91 }),
92 );
93 const collected = `${collector.url}/collected`;
94 yield* expectUrlContains(collected, "otel-collector-ok", { timeout: "180 seconds" });
95
96 // The fixture's telemetry binding layer reads `COLLECTOR_URL` via
97 // `Config` at Init. The test harness snapshots its ConfigProvider
98 // before the test body runs, so layer the just-learned collector
99 // URL on top of the current provider for this deploy only.
100 const currentConfig = yield* ConfigProvider.ConfigProvider;
101 const { fn } = yield* stack
102 .deploy(
103 Effect.gen(function* () {
104 yield* collectorWorker();
105 const fn = yield* OtelTestFunction.pipe(Effect.provide(OtelTestFunctionLive));
106 return { fn };
107 }),
108 )
109 .pipe(
110 Effect.provideService(
111 ConfigProvider.ConfigProvider,
112 ConfigProvider.orElse(
113 ConfigProvider.fromUnknown({ COLLECTOR_URL: collector.url }),
114 currentConfig,
115 ),
116 ),
117 );
118
119 // Wait for the function URL to serve AND for the collector to be
120 // reachable from the Lambda's region — workers.dev propagates
121 // per-PoP, so the Lambda can see placeholder 404s long after the
122 // test machine gets 200s.
123 const fnUrl = (fn.functionUrl as string).replace(/\/$/, "");
124 yield* expectUrlContains(`${fnUrl}/probe`, '"status":200', {
125 timeout: "240 seconds",
126 label: "collector reachable from Lambda",
127 });
128
129 // Drive one traced invocation.
130 yield* expectUrlContains(`${fnUrl}/work`, "lambda-did-work", { timeout: "120 seconds" });
131
132 // The invocation scope's flush ships everything the invocation
133 // produced before the response is returned.
134 yield* expectUrlContains(collected, "otel-lambda-test", {
135 timeout: "120 seconds",
136 label: "lambda service.name",
137 });
138 // HTTP server root span from the HttpMiddleware tracer, named
139 // after the request method.
140 yield* expectUrlContains(collected, '"name":"GET"');
141 // Child span from Effect.fn instrumentation.
142 yield* expectUrlContains(collected, "lambda.child-span");
143 // Log record shipped by the OTLP logger.
144 yield* expectUrlContains(collected, "lambda-work-log");
145
146 // An invocation that runs past its 3 s timeout is interrupted:
147 // its finalizer runs, the span of the call that hung ends, and the
148 // root span ends with the timeout — all exported. The invocation
149 // is then held until Lambda's own 5 s timeout ends it, so the
150 // caller sees Lambda's timeout error, not a response from the
151 // handler.
152 const client = yield* HttpClient.HttpClient;
153 const slowStart = yield* Clock.currentTimeMillis;
154 const slow = yield* client.get(`${fnUrl}/slow`);
155 const slowElapsed = (yield* Clock.currentTimeMillis) - slowStart;
156 expect(slow.status).not.toBe(200);
157 expect(slowElapsed).toBeGreaterThanOrEqual(4500);
158
159 // Lambda counts it as a real timeout and shuts the sandbox down
160 // (our shutdown extension logs the SIGTERM inside the invocation),
161 // so work that ignored the interruption cannot resume later.
162 const logGroupName = `/aws/lambda/${fn.functionName}`;
163 const slowRequestLines = yield* logs.filterLogEvents({ logGroupName }).pipe(
164 Effect.map((result) => (result.events ?? []).map((event) => event.message ?? "")),
165 Effect.map((lines) => {
166 const requestId = lines
167 .map((line) => /Lambda invocation (\S+) of .* timed out after/.exec(line)?.[1])
168 .find((id) => id !== undefined);
169 return requestId === undefined
170 ? []
171 : lines.filter((line) => line.includes(requestId));
172 }),
173 Effect.repeat({
174 schedule: Schedule.spaced("2 seconds"),
175 until: (lines) => lines.some((line) => line.startsWith("REPORT")),
176 times: 30,
177 }),
178 );
179 expect(slowRequestLines.find((line) => line.startsWith("REPORT"))).toContain(
180 "Status: timeout",
181 );
182 expect(slowRequestLines.some((line) => line.includes("SIGTERM"))).toBe(true);
183 yield* expectUrlContains(collected, "lambda-slow-finalizer", {
184 timeout: "120 seconds",
185 label: "interrupted invocation's finalizer log",
186 });
187 yield* expectUrlContains(collected, "lambda-slow-log");
188 const slowBody = yield* client
189 .get(collected)
190 .pipe(Effect.flatMap((response) => response.json));
191 const slowRoots = rootSpansFor(slowBody, "/slow");
192 expect({ slowRoots: slowRoots.length, roots: rootSpanSummary(slowBody) }).toMatchObject({
193 slowRoots: 1,
194 });
195 expect(slowRoots[0]?.status).toMatchObject({
196 code: 2,
197 message: expect.stringContaining("timed out"),
198 });
199 // The span of the call that hung, ended by the interruption.
200 const hung = collectedSpans(slowBody).filter((span) => span.name === "lambda.slow-span");
201 expect(hung).toHaveLength(1);
202 expect(hung[0]?.attributes).toContainEqual({
203 key: "effect.fiber.interrupted",
204 value: { boolValue: true },
205 });
206
207 // An invocation that finishes just inside its timeout is never
208 // interrupted, and its root span is a plain success.
209 const late = yield* client.get(`${fnUrl}/late`);
210 expect(late.status).toBe(200);
211 expect(yield* late.text).toContain("lambda-late-done");
212 const lateBody = yield* client
213 .get(collected)
214 .pipe(Effect.flatMap((response) => response.json));
215 const lateSpans = rootSpansFor(lateBody, "/late");
216 expect({ lateSpans: lateSpans.length, roots: rootSpanSummary(lateBody) }).toMatchObject({
217 lateSpans: 1,
218 });
219 expect(lateSpans[0]?.status?.code).not.toBe(2);
220
221 // An invocation whose finalizer hangs past Lambda's own timeout:
222 // Lambda kills it, but 500 ms before that the backstop flush ended
223 // the root span and drained the exporters.
224 const stuck = yield* client.get(`${fnUrl}/stuck`);
225 expect(stuck.status).not.toBe(200);
226 const stuckSpans = yield* client.get(collected).pipe(
227 Effect.flatMap((response) => response.json),
228 Effect.map((body) => rootSpansFor(body, "/stuck")),
229 Effect.repeat({
230 schedule: Schedule.spaced("2 seconds"),
231 until: (spans) => spans.length > 0,
232 times: 30,
233 }),
234 );
235 expect(stuckSpans).toHaveLength(1);
236 expect(stuckSpans[0]?.status?.code).toBe(2);
237 expect(stuckSpans[0]?.attributes).toContainEqual({
238 key: "aws.lambda.timeout.imminent",
239 value: { boolValue: true },
240 });
241 }),
242 { timeout: 600_000 },
243 );
244 },
245 );