@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 813 lines 29.3 KB
Raw
1 import * as Cloudflare from "@alchemy.run/cloudflare";
2 import { CloudflareEnvironment } from "@alchemy.run/cloudflare/CloudflareEnvironment";
3 import {
4 CloudflareTelemetryCompatibilityError,
5 MIN_CLOUDFLARE_TRACING_DATE,
6 } from "@alchemy.run/cloudflare/Workers/Telemetry";
7 import type { Worker } from "@alchemy.run/cloudflare/Workers/Worker";
8 import { resolveObservability } from "@alchemy.run/cloudflare/Workers/WorkerAsyncBindings";
9 import { describe, expect, test as unit } from "@alchemy.run/test";
10 import * as Test from "@alchemy.run/test/Alchemy";
11 import * as workers from "@distilled.cloud/cloudflare/workers";
12 import type { ResourceBinding } from "alchemy/Resource";
13 import * as ConfigProvider from "effect/ConfigProvider";
14 import * as Effect from "effect/Effect";
15 import * as HttpClient from "effect/http/HttpClient";
16 import { MinimumLogLevel } from "effect/References";
17 import * as Schedule from "effect/Schedule";
18 import * as pathe from "pathe";
19 import { stageSuffix } from "../../names.ts";
20 import { expectUrlContains } from "../Utils/Http.ts";
21 import { waitForWorkerToBeDeleted } from "../Utils/Worker.ts";
22 import type { TraceSink } from "./fixtures/native-tracing/collector.ts";
23 import NativeTracingWorker, { makeTracedWorker } from "./fixtures/native-tracing/worker.ts";
24
25 const { test } = Test.make({ providers: Cloudflare.providers() });
26
27 const logLevel = Effect.provideService(MinimumLogLevel, process.env.DEBUG ? "Debug" : "Info");
28
29 /**
30 * A span row Workers Observability ingested: Cloudflare's own
31 * `traceId` / `spanId` / `parentSpanId` (so the tree the platform recorded
32 * can be checked directly) plus the `effect.exit` attribute the tracer
33 * forwards on span end.
34 */
35 interface SpanRow extends workers.ObservabilitySharedQueriesGetResponseEventsEventsItemMetadata {
36 exit: unknown;
37 }
38
39 const exitOf = (source: unknown): unknown => {
40 if (typeof source !== "object" || source === null) return undefined;
41 const effect = (source as Record<string, unknown>).effect;
42 return typeof effect === "object" && effect !== null
43 ? (effect as Record<string, unknown>).exit
44 : undefined;
45 };
46
47 const querySpans = (accountId: string, service: string) =>
48 // The timeframe is re-evaluated per poll: rows are keyed by ingestion
49 // time, so a window fixed at the first attempt never sees late arrivals.
50 Effect.suspend(() =>
51 workers.queryObservabilityTelemetry({
52 accountId,
53 queryId: "adhoc-native-tracing",
54 view: "events",
55 timeframe: { from: Date.now() - 15 * 60 * 1000, to: Date.now() },
56 limit: 500,
57 parameters: {
58 filters: [
59 {
60 key: "$metadata.service",
61 operation: "eq",
62 type: "string",
63 value: service,
64 },
65 { key: "$metadata.spanName", operation: "exists", type: "string" },
66 ],
67 },
68 }),
69 ).pipe(
70 Effect.map((response) =>
71 (response.events?.events ?? []).map((event): SpanRow => ({
72 ...event.metadata,
73 exit: exitOf(event.source),
74 })),
75 ),
76 );
77
78 /** The spans of one request: the root matched by name + `request.id`, and every row sharing its trace. */
79 const traceOf = (spans: SpanRow[], root: string, id: string) => {
80 const rootSpan = spans.find((s) => s.spanName === root && s.requestId === id);
81 return rootSpan === undefined
82 ? undefined
83 : {
84 root: rootSpan,
85 spans: spans.filter((s) => s.traceId === rootSpan.traceId),
86 /** Exactly one span of this name in the trace. */
87 span: (name: string) => {
88 const matches = spans.filter(
89 (s) => s.traceId === rootSpan.traceId && s.spanName === name,
90 );
91 expect(matches).toHaveLength(1);
92 return matches[0]!;
93 },
94 };
95 };
96
97 type Bindings = ReadonlyArray<ResourceBinding<Worker["Binding"]>>;
98
99 const tracesBind = (traces: NonNullable<Worker["Binding"]["observability"]>["traces"]): Bindings =>
100 [{ sid: "Cloudflare.Telemetry", data: { observability: { traces } } }] as unknown as Bindings;
101
102 describe(
103 "resolveObservability",
104 {
105 tags: ["unit", "provider:cloudflare", "provider:cloudflare:worker", "local"],
106 },
107 () => {
108 unit("omitted props + bound traces keep default logs", () => {
109 const resolved = resolveObservability({}, tracesBind({ enabled: true }));
110 expect(resolved.enabled).toBe(true);
111 expect(resolved.logs?.enabled).toBe(true);
112 expect(resolved.logs?.invocationLogs).toBe(true);
113 expect(resolved.traces?.enabled).toBe(true);
114 });
115
116 unit("explicit traces.enabled false wins over the bind", () => {
117 const resolved = resolveObservability(
118 { observability: { enabled: true, traces: { enabled: false } } },
119 tracesBind({ enabled: true, persist: true }),
120 );
121 expect(resolved.traces?.enabled).toBe(false);
122 expect(resolved.traces?.persist).toBeUndefined();
123 });
124
125 unit("fills traces when news.observability exists without traces", () => {
126 const resolved = resolveObservability(
127 {
128 observability: {
129 enabled: true,
130 logs: { enabled: true, invocationLogs: true, persist: true },
131 },
132 },
133 tracesBind({ enabled: true, headSamplingRate: 1 }),
134 );
135 expect(resolved.logs?.persist).toBe(true);
136 expect(resolved.traces?.enabled).toBe(true);
137 expect(resolved.traces?.headSamplingRate).toBe(1);
138 });
139
140 unit("Issues coexist with Telemetry traces and explicit logs", () => {
141 const resolved = resolveObservability(
142 {
143 observability: {
144 enabled: true,
145 logs: { enabled: true, invocationLogs: true },
146 issues: { enabled: true },
147 },
148 },
149 tracesBind({ enabled: true, headSamplingRate: 0.1 }),
150 );
151 expect(resolved.issues?.enabled).toBe(true);
152 expect(resolved.logs?.invocationLogs).toBe(true);
153 expect(resolved.traces?.enabled).toBe(true);
154 expect(resolved.traces?.headSamplingRate).toBe(0.1);
155 });
156
157 unit("no bind returns default logs", () => {
158 const resolved = resolveObservability({}, []);
159 expect(resolved.traces).toBeUndefined();
160 expect(resolved.logs?.invocationLogs).toBe(true);
161 });
162 },
163 );
164
165 test.provider(
166 "Cloudflare.Telemetry() enables traces without clobbering default logs",
167 (stack) =>
168 Effect.gen(function* () {
169 const { accountId } = yield* yield* CloudflareEnvironment;
170 yield* stack.destroy();
171
172 const worker = yield* stack.deploy(
173 Effect.gen(function* () {
174 return yield* NativeTracingWorker;
175 }),
176 );
177
178 const settings = yield* workers.getScriptScriptAndVersionSetting({
179 accountId,
180 scriptName: worker.workerName,
181 });
182 expect(settings.observability?.enabled).toBe(true);
183 expect(settings.observability?.logs?.enabled).toBe(true);
184 expect(settings.observability?.logs?.invocationLogs).toBe(true);
185 expect(settings.observability?.traces?.enabled).toBe(true);
186
187 yield* expectUrlContains(`${worker.url}/work`, "native-did-work", {
188 timeout: "180 seconds",
189 });
190
191 yield* stack.destroy();
192 yield* waitForWorkerToBeDeleted(worker.workerName, accountId);
193 }).pipe(logLevel),
194 {
195 tags: [
196 "provider:cloudflare",
197 "provider:cloudflare:kv",
198 "provider:cloudflare:queue",
199 "provider:cloudflare:worker",
200 "live",
201 ],
202 timeout: 180_000,
203 },
204 );
205
206 test.provider(
207 "explicit observability.traces wins over Cloudflare.Telemetry() bind",
208 (stack) =>
209 Effect.gen(function* () {
210 const { accountId } = yield* yield* CloudflareEnvironment;
211 yield* stack.destroy();
212
213 const worker = yield* stack.deploy(
214 Effect.gen(function* () {
215 return yield* makeTracedWorker("NativeTracingOverride", {
216 observability: {
217 enabled: true,
218 traces: { enabled: false },
219 },
220 });
221 }),
222 );
223
224 const settings = yield* workers.getScriptScriptAndVersionSetting({
225 accountId,
226 scriptName: worker.workerName,
227 });
228 expect(settings.observability?.traces?.enabled).toBe(false);
229 expect(settings.observability?.logs?.invocationLogs).toBe(true);
230
231 yield* stack.destroy();
232 yield* waitForWorkerToBeDeleted(worker.workerName, accountId);
233 }).pipe(logLevel),
234 {
235 tags: [
236 "provider:cloudflare",
237 "provider:cloudflare:kv",
238 "provider:cloudflare:queue",
239 "provider:cloudflare:worker",
240 "live",
241 ],
242 timeout: 180_000,
243 },
244 );
245
246 test.provider(
247 "Cloudflare.Telemetry() fails deploy on a pre-startActiveSpan compatibility date",
248 (stack) =>
249 Effect.gen(function* () {
250 yield* stack.destroy();
251
252 const error = yield* stack
253 .deploy(
254 Effect.gen(function* () {
255 return yield* makeTracedWorker("NativeTracingOldDate", {
256 compatibility: { date: "2026-03-17" },
257 });
258 }),
259 )
260 .pipe(Effect.flip);
261
262 expect(error).toBeInstanceOf(CloudflareTelemetryCompatibilityError);
263 expect(String(error)).toContain(MIN_CLOUDFLARE_TRACING_DATE);
264 expect(String(error)).toContain("2026-03-17");
265
266 yield* stack.destroy();
267 }).pipe(logLevel),
268 {
269 tags: [
270 "provider:cloudflare",
271 "provider:cloudflare:kv",
272 "provider:cloudflare:queue",
273 "provider:cloudflare:worker",
274 "live",
275 ],
276 timeout: 120_000,
277 },
278 );
279
280 test.provider(
281 "Effect spans appear in Workers Observability after Cloudflare.Telemetry()",
282 (stack) =>
283 Effect.gen(function* () {
284 const { accountId } = yield* yield* CloudflareEnvironment;
285 yield* stack.destroy();
286
287 const worker = yield* stack.deploy(
288 Effect.gen(function* () {
289 return yield* NativeTracingWorker;
290 }),
291 );
292
293 yield* expectUrlContains(`${worker.url}/work`, "native-did-work", {
294 timeout: "180 seconds",
295 });
296
297 // Poll until the request's Effect spans have been ingested.
298 const spans = yield* querySpans(accountId, worker.workerName).pipe(
299 Effect.repeat({
300 schedule: Schedule.spaced("5 seconds"),
301 until: (spans) =>
302 spans.some((s) => s.spanName === "operation") &&
303 spans.some((s) => s.spanName === "native.child"),
304 times: 36,
305 }),
306 );
307 expect(spans.map((s) => s.spanName)).toEqual(
308 expect.arrayContaining(["operation", "native.child"]),
309 );
310
311 yield* stack.destroy();
312 yield* waitForWorkerToBeDeleted(worker.workerName, accountId);
313 }).pipe(logLevel),
314 {
315 tags: [
316 "provider:cloudflare",
317 "provider:cloudflare:kv",
318 "provider:cloudflare:queue",
319 "provider:cloudflare:worker",
320 "live",
321 ],
322 timeout: 300_000,
323 },
324 );
325
326 const FANOUT_SPANS = [
327 "operation",
328 "child.a",
329 "child.a.inner",
330 "child.b",
331 "child.b.inner",
332 "forked",
333 "forked.inner",
334 ];
335
336 test.provider(
337 "records the Effect span tree across fibers, exits, Durable Object and queue handlers",
338 (stack) =>
339 Effect.gen(function* () {
340 const { accountId } = yield* yield* CloudflareEnvironment;
341 yield* stack.destroy();
342
343 const worker = yield* stack.deploy(
344 Effect.gen(function* () {
345 return yield* NativeTracingWorker;
346 }),
347 );
348 yield* expectUrlContains(`${worker.url}/fanout?id=warmup`, "native-did-fanout", {
349 timeout: "180 seconds",
350 });
351
352 // Each request retries through workers.dev propagation (a colo can
353 // still serve Cloudflare's error page right after the warm-up
354 // succeeded elsewhere); a retried id just produces one more
355 // independent trace.
356 const hit = (path: string, id: string, marker = id) =>
357 expectUrlContains(`${worker.url}${path}?id=${id}`, marker, {
358 timeout: "120 seconds",
359 });
360 // One request shape per event path. A request's trace is located by
361 // the `request.id` annotated on its root span (`$metadata.requestId`)
362 // and is complete once every expected span arrived, plus a platform
363 // span (e.g. the KV read) under each span in `platformUnder`.
364 const shapes = [
365 {
366 name: "fanout",
367 root: "operation",
368 names: FANOUT_SPANS,
369 platformUnder: ["child.a", "child.b", "forked"],
370 // Several in flight at once so fibers from different requests
371 // interleave inside the isolate.
372 send: (id: string) => hit("/fanout", id),
373 concurrent: 3,
374 },
375 {
376 name: "exits",
377 root: "operation",
378 names: ["operation", "failing.child", "interrupted.child"],
379 platformUnder: [],
380 send: (id: string) => hit("/exits", id),
381 concurrent: 1,
382 },
383 {
384 name: "rpc",
385 root: "do.operation",
386 names: ["do.operation", "do.inner"],
387 platformUnder: [],
388 send: (id: string) => hit("/rpc", id, "native-did-rpc:do-ok"),
389 concurrent: 1,
390 },
391 {
392 name: "queue",
393 root: "queue.operation",
394 names: ["queue.operation", "queue.inner"],
395 platformUnder: [],
396 send: (id: string) => hit("/enqueue", id),
397 concurrent: 1,
398 },
399 ] as const;
400 type Shape = (typeof shapes)[number];
401 const complete = (spans: SpanRow[], shape: Shape, id: string) => {
402 const trace = traceOf(spans, shape.root, id);
403 return (
404 trace !== undefined &&
405 shape.names.every((name) => trace.spans.some((s) => s.spanName === name)) &&
406 shape.platformUnder.every((name) =>
407 trace.spans.some(
408 (parent) =>
409 parent.spanName === name &&
410 trace.spans.some(
411 (s) =>
412 s.parentSpanId === parent.spanId && !FANOUT_SPANS.includes(s.spanName ?? ""),
413 ),
414 ),
415 )
416 );
417 };
418 // Every id sent per shape, so a trace that arrives late still counts.
419 const sent = new Map<Shape, string[]>(shapes.map((shape) => [shape, []]));
420 const send = (shape: Shape) =>
421 Effect.gen(function* () {
422 const ids = yield* Effect.sync(() => {
423 const ids = Array.from({ length: shape.concurrent }, () =>
424 globalThis.crypto.randomUUID(),
425 );
426 sent.get(shape)!.push(...ids);
427 return ids;
428 });
429 yield* Effect.all(ids.map(shape.send), { concurrency: "unbounded" });
430 });
431 yield* Effect.all(
432 [
433 ...shapes.map(send),
434 // Head sampling 1: every Effect span is sampled.
435 expectUrlContains(`${worker.url}/sampled`, '"operation":true,"child":true', {
436 timeout: "120 seconds",
437 }),
438 ],
439 { concurrency: "unbounded" },
440 );
441
442 // Ingestion latency varies from seconds to several minutes, and
443 // ingestion is lossy: a span and the platform spans under it can go
444 // missing for good, so waiting on one request can never finish. Every
445 // minute, send a fresh request for each shape that has no complete
446 // trace yet; the assertions below run on the traces that arrived whole.
447 const completeIds = (spans: SpanRow[], shape: Shape) =>
448 sent.get(shape)!.filter((id) => complete(spans, shape, id));
449 let polls = 0;
450 const poll = Effect.gen(function* () {
451 const spans = yield* querySpans(accountId, worker.workerName);
452 const missing = shapes.filter((shape) => completeIds(spans, shape).length === 0);
453 const count = yield* Effect.sync(() => ++polls);
454 if (missing.length > 0 && count % 12 === 0) {
455 yield* Effect.logInfo(
456 `native-tracing: no complete trace yet for ${missing.map((s) => s.name).join(", ")}; sending again`,
457 );
458 yield* Effect.all(missing.map(send), { concurrency: "unbounded" });
459 }
460 return { spans, missing };
461 });
462 const { spans, missing } = yield* poll.pipe(
463 Effect.repeat({
464 schedule: Schedule.spaced("5 seconds"),
465 until: (result) => result.missing.length === 0,
466 times: 60,
467 }),
468 );
469 if (missing.length > 0) {
470 yield* Effect.logError(
471 `native-tracing: no complete trace for ${missing.map((s) => s.name).join(", ")}; spans ingested so far: ${JSON.stringify(
472 spans.map((s) => [s.requestId, s.traceId, s.spanName]),
473 )}`,
474 );
475 }
476 expect(missing.map((shape) => shape.name)).toEqual([]);
477 const [fanout, exits, rpc, queue] = shapes;
478 const ids = {
479 fanout: completeIds(spans, fanout),
480 exits: completeIds(spans, exits)[0]!,
481 rpc: completeIds(spans, rpc)[0]!,
482 queue: completeIds(spans, queue)[0]!,
483 };
484
485 // Fan-out: siblings and nesting follow the Effect span tree, and each
486 // KV read (a Cloudflare auto-instrumented span) is attributed to the
487 // branch whose fiber issued it — even though the scheduler resumed
488 // that fiber on a timer tick outside `operation`'s async context.
489 for (const id of ids.fanout) {
490 const trace = traceOf(spans, "operation", id)!;
491 const operation = trace.root;
492 // `operation` hangs off Cloudflare's own request span.
493 expect(
494 trace.spans.some(
495 (s) => s.spanId === operation.parentSpanId && s.spanName !== "operation",
496 ),
497 ).toBe(true);
498 for (const name of ["child.a", "child.b", "forked"]) {
499 const branch = trace.span(name);
500 expect(branch.parentSpanId).toBe(operation.spanId);
501 expect(trace.span(`${name}.inner`).parentSpanId).toBe(branch.spanId);
502 const platform = trace.spans.filter(
503 (s) => s.parentSpanId === branch.spanId && !FANOUT_SPANS.includes(s.spanName ?? ""),
504 );
505 expect(platform.length).toBeGreaterThanOrEqual(1);
506 }
507 expect(operation.exit).toBe("success");
508 }
509
510 // Exits: the failing child (caught by its parent) and the interrupted
511 // child record their outcome as `effect.exit`.
512 {
513 const trace = traceOf(spans, "operation", ids.exits)!;
514 expect(trace.root.exit).toBe("success");
515 const failing = trace.span("failing.child");
516 expect(failing.parentSpanId).toBe(trace.root.spanId);
517 expect(failing.exit).toBe("failure");
518 const interrupted = trace.span("interrupted.child");
519 expect(interrupted.parentSpanId).toBe(trace.root.spanId);
520 expect(interrupted.exit).toBe("interrupted");
521 }
522
523 // Durable Object RPC: spans opened inside the DO method nest under
524 // the DO invocation's platform span.
525 {
526 const trace = traceOf(spans, "do.operation", ids.rpc)!;
527 expect(trace.root.parentSpanId).toBeTruthy();
528 expect(trace.span("do.inner").parentSpanId).toBe(trace.root.spanId);
529 expect(trace.root.exit).toBe("success");
530 }
531
532 // Queue consumer: same for the `queue` event handler.
533 {
534 const trace = traceOf(spans, "queue.operation", ids.queue)!;
535 expect(trace.root.parentSpanId).toBeTruthy();
536 expect(trace.span("queue.inner").parentSpanId).toBe(trace.root.spanId);
537 expect(trace.root.exit).toBe("success");
538 }
539
540 yield* stack.destroy();
541 yield* waitForWorkerToBeDeleted(worker.workerName, accountId);
542 }).pipe(logLevel),
543 {
544 tags: [
545 "provider:cloudflare",
546 "provider:cloudflare:kv",
547 "provider:cloudflare:queue",
548 "provider:cloudflare:worker",
549 "live",
550 ],
551 timeout: 480_000,
552 },
553 );
554
555 // The OTLP collector the export test's observability destination pushes
556 // to. Cloudflare's exporter needs a public hostname, so the collector
557 // gets a deterministic custom domain on the standing test zone.
558 const zoneName = process.env.CLOUDFLARE_TEST_DNS_ZONE_NAME ?? "alchemy-test-2.us";
559 const collectorHost = `native-trace-collector-${stageSuffix()}.${zoneName}`;
560 // Named up front: the traced Worker lists the destination by name.
561 const destinationName = `native-trace-export-${stageSuffix()}`;
562 const collectorMain = pathe.resolve(import.meta.dirname, "fixtures/native-tracing/collector.ts");
563
564 /** An exported OTLP span, reduced to what the status assertions read. */
565 interface ExportedSpan {
566 traceId: string;
567 name: string;
568 requestId: unknown;
569 status: { code: "unset" | "ok" | "error"; message?: string };
570 }
571
572 // OTLP/JSON encodes the status code as its enum number or its name.
573 const statusCode = (code: unknown): ExportedSpan["status"]["code"] =>
574 code === 2 || code === "STATUS_CODE_ERROR"
575 ? "error"
576 : code === 1 || code === "STATUS_CODE_OK"
577 ? "ok"
578 : "unset";
579
580 // The destination's preflight POST stores a non-OTLP body too.
581 const parseJson = (text: string): any => {
582 try {
583 return JSON.parse(text);
584 } catch {
585 return undefined;
586 }
587 };
588
589 const exportedSpans = (items: ReadonlyArray<string>): ExportedSpan[] =>
590 items.flatMap((item) => {
591 const body = parseJson(item);
592 return (body?.resourceSpans ?? []).flatMap((resource: any) =>
593 (resource.scopeSpans ?? []).flatMap((scope: any) =>
594 (scope.spans ?? []).map((span: any): ExportedSpan => ({
595 traceId: span.traceId,
596 name: span.name,
597 requestId: (span.attributes ?? []).find((a: any) => a.key === "request.id")?.value
598 ?.stringValue,
599 status: { code: statusCode(span.status?.code), message: span.status?.message },
600 })),
601 ),
602 );
603 });
604
605 test.provider(
606 "exports a failed Effect span with error status over OTLP",
607 (stack) =>
608 Effect.gen(function* () {
609 const { accountId } = yield* yield* CloudflareEnvironment;
610 yield* stack.destroy();
611
612 // The traced Worker lists the destination by name, so the collector
613 // and destination deploy first and stay declared in the second step.
614 const sink = Effect.gen(function* () {
615 const collector = yield* Cloudflare.Worker("NativeTraceCollector", {
616 main: collectorMain,
617 env: { SINK: Cloudflare.DurableObject<TraceSink>("TraceSink") },
618 domain: collectorHost,
619 compatibility: { date: "2024-09-23" },
620 });
621 yield* Cloudflare.Workers.ObservabilityDestination("NativeTraceDestination", {
622 name: destinationName,
623 url: `https://${collectorHost}/v1/traces`,
624 logpushDataset: "opentelemetry-traces",
625 // The collector's custom domain may not serve yet on create.
626 skipPreflightCheck: true,
627 });
628 return collector;
629 });
630 yield* stack.deploy(sink);
631 const { collector, worker } = yield* stack.deploy(
632 Effect.gen(function* () {
633 const collector = yield* sink;
634 const worker = yield* makeTracedWorker("NativeTracingExport", {
635 observability: {
636 enabled: true,
637 traces: { enabled: true, destinations: [destinationName] },
638 },
639 });
640 return { collector, worker };
641 }),
642 );
643
644 // Read back over workers.dev: a custom domain attached makes
645 // `collector.url` the domain, so derive the workers.dev URL.
646 const { subdomain } = yield* workers.getSubdomain({ accountId });
647 const collected = `https://${collector.workerName}.${subdomain}.workers.dev/collected`;
648 yield* expectUrlContains(collected, "trace-collector-ok", { timeout: "180 seconds" });
649 yield* expectUrlContains(`${worker.url}/exits?id=warmup`, "native-did-exits", {
650 timeout: "180 seconds",
651 });
652
653 // Export runs through Logpush batches, so a request's spans can take
654 // minutes to arrive. Send a fresh `/exits` request every minute until
655 // one request's whole trace has been exported.
656 const client = yield* HttpClient.HttpClient;
657 const sent: string[] = [];
658 const send = Effect.gen(function* () {
659 const id = yield* Effect.sync(() => {
660 const id = globalThis.crypto.randomUUID();
661 sent.push(id);
662 return id;
663 });
664 yield* expectUrlContains(`${worker.url}/exits?id=${id}`, id, { timeout: "120 seconds" });
665 });
666 const traceFor = (spans: ExportedSpan[]) =>
667 sent
668 .map((id) => {
669 const root = spans.find((s) => s.name === "operation" && s.requestId === id);
670 const trace = spans.filter((s) => s.traceId === root?.traceId);
671 const span = (name: string) => trace.find((s) => s.name === name);
672 return root && span("failing.child") && span("interrupted.child")
673 ? { root, failing: span("failing.child")!, interrupted: span("interrupted.child")! }
674 : undefined;
675 })
676 .find((trace) => trace !== undefined);
677 let polls = 0;
678 const trace = yield* Effect.gen(function* () {
679 const count = yield* Effect.sync(() => polls++);
680 if (count % 12 === 0) yield* send;
681 const body = (yield* client.get(collected).pipe(Effect.flatMap((res) => res.json))) as {
682 items: string[];
683 };
684 return traceFor(exportedSpans(body.items));
685 }).pipe(
686 Effect.repeat({
687 schedule: Schedule.spaced("5 seconds"),
688 until: (trace) => trace !== undefined,
689 times: 72,
690 }),
691 );
692
693 expect(trace).toBeDefined();
694 // A failure maps to the standard error status; success and pure
695 // interruption stay unset.
696 expect(trace!.failing.status).toEqual({ code: "error", message: "boom" });
697 expect(trace!.interrupted.status.code).toBe("unset");
698 expect(trace!.root.status.code).toBe("unset");
699
700 yield* stack.destroy();
701 yield* waitForWorkerToBeDeleted(worker.workerName, accountId);
702 }).pipe(logLevel),
703 {
704 tags: [
705 "provider:cloudflare",
706 "provider:cloudflare:kv",
707 "provider:cloudflare:queue",
708 "provider:cloudflare:worker",
709 "live",
710 ],
711 timeout: 480_000,
712 },
713 );
714
715 test.provider(
716 "head sampling below 1 is applied by Cloudflare at ingestion",
717 (stack) =>
718 Effect.gen(function* () {
719 const { accountId } = yield* yield* CloudflareEnvironment;
720 const currentConfig = yield* ConfigProvider.ConfigProvider;
721 yield* stack.destroy();
722
723 const worker = yield* stack
724 .deploy(
725 Effect.gen(function* () {
726 return yield* makeTracedWorker("NativeTracingSampled");
727 }),
728 )
729 .pipe(
730 Effect.provideService(
731 ConfigProvider.ConfigProvider,
732 ConfigProvider.orElse(
733 ConfigProvider.fromUnknown({ HEAD_SAMPLING_RATE: 0.5 }),
734 currentConfig,
735 ),
736 ),
737 );
738
739 const settings = yield* workers.getScriptScriptAndVersionSetting({
740 accountId,
741 scriptName: worker.workerName,
742 });
743 expect(settings.observability?.traces?.enabled).toBe(true);
744 expect(settings.observability?.traces?.headSamplingRate).toBe(0.5);
745
746 // The runtime reports every invocation as traced (`span.isTraced` does
747 // not carry the head-sampling decision — a rate of 0 behaves the same),
748 // so Effect's `sampled` flag stays true and consistent parent → child.
749 // Sampling is applied when Cloudflare ingests: of N invocations, only
750 // ~half persist.
751 //
752 // A 50% rate (rather than a few percent) keeps this test fast and
753 // two-sided: some request of the batch is always persisted, so the
754 // ingestion wait ends as soon as rows land instead of at its ceiling,
755 // and the count can be bounded from below as well as above.
756 const runId = yield* Effect.sync(() => globalThis.crypto.randomUUID());
757 yield* expectUrlContains(`${worker.url}/sampled?id=warmup-${runId}`, "native-did-sample", {
758 timeout: "180 seconds",
759 });
760 const client = yield* HttpClient.HttpClient;
761 const probe = client.get(`${worker.url}/sampled?id=${runId}`).pipe(
762 Effect.flatMap((res) => res.json),
763 Effect.map((body) => body as { operation: boolean; child: boolean }),
764 Effect.retry({ schedule: Schedule.spaced("1 second"), times: 30 }),
765 );
766 const requests = 100;
767 const seen = yield* Effect.all(
768 Array.from({ length: requests }, () => probe),
769 { concurrency: 10 },
770 );
771 expect(seen.filter((b) => b.child !== b.operation)).toEqual([]);
772
773 // One `operation` root per persisted request, keyed by this run's id
774 // so earlier runs inside the 15-minute query window don't count.
775 // Rows land in batches, so wait until two consecutive reads agree
776 // (and enough exist to prove tracing is on) before bounding the count.
777 // Binomial(100, 0.5) is within [20, 80] with P(miss) ≈ 1e-9; a few
778 // retried probes add at most a handful of extra traces.
779 const persistedCount = querySpans(accountId, worker.workerName).pipe(
780 Effect.map(
781 (spans) =>
782 spans.filter((s) => s.spanName === "operation" && s.requestId === runId).length,
783 ),
784 );
785 let previous = -1;
786 const persisted = yield* persistedCount.pipe(
787 Effect.repeat({
788 schedule: Schedule.spaced("5 seconds"),
789 until: (count) => {
790 const settled = count >= 20 && count === previous;
791 previous = count;
792 return settled;
793 },
794 times: 36,
795 }),
796 );
797 expect(persisted).toBeGreaterThanOrEqual(20);
798 expect(persisted).toBeLessThanOrEqual(80);
799
800 yield* stack.destroy();
801 yield* waitForWorkerToBeDeleted(worker.workerName, accountId);
802 }).pipe(logLevel),
803 {
804 tags: [
805 "provider:cloudflare",
806 "provider:cloudflare:kv",
807 "provider:cloudflare:queue",
808 "provider:cloudflare:worker",
809 "live",
810 ],
811 timeout: 300_000,
812 },
813 );