The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 06:00 UTC
| 1 | import * as Cloudflare from "@alchemy.run/cloudflare"; |
| 2 | import { CloudflareEnvironment } from "@alchemy.run/cloudflare/CloudflareEnvironment"; |
| 3 | import { |
| 4 | CloudflareTelemetryCompatibilityError, |
| 5 | MIN_CLOUDFLARE_TRACING_DATE, |
| 6 | } from "@alchemy.run/cloudflare/Workers/Telemetry"; |
| 7 | import type { Worker } from "@alchemy.run/cloudflare/Workers/Worker"; |
| 8 | import { resolveObservability } from "@alchemy.run/cloudflare/Workers/WorkerAsyncBindings"; |
| 9 | import { describe, expect, test as unit } from "@alchemy.run/test"; |
| 10 | import * as Test from "@alchemy.run/test/Alchemy"; |
| 11 | import * as workers from "@distilled.cloud/cloudflare/workers"; |
| 12 | import type { ResourceBinding } from "alchemy/Resource"; |
| 13 | import * as ConfigProvider from "effect/ConfigProvider"; |
| 14 | import * as Effect from "effect/Effect"; |
| 15 | import * as HttpClient from "effect/http/HttpClient"; |
| 16 | import { MinimumLogLevel } from "effect/References"; |
| 17 | import * as Schedule from "effect/Schedule"; |
| 18 | import * as pathe from "pathe"; |
| 19 | import { stageSuffix } from "../../names.ts"; |
| 20 | import { expectUrlContains } from "../Utils/Http.ts"; |
| 21 | import { waitForWorkerToBeDeleted } from "../Utils/Worker.ts"; |
| 22 | import type { TraceSink } from "./fixtures/native-tracing/collector.ts"; |
| 23 | import NativeTracingWorker, { makeTracedWorker } from "./fixtures/native-tracing/worker.ts"; |
| 24 | |
| 25 | const { test } = Test.make({ providers: Cloudflare.providers() }); |
| 26 | |
| 27 | const logLevel = Effect.provideService(MinimumLogLevel, process.env.DEBUG ? "Debug" : "Info"); |
| 28 | |
| 29 | /** |
| 30 | * A span row Workers Observability ingested: Cloudflare's own |
| 31 | * `traceId` / `spanId` / `parentSpanId` (so the tree the platform recorded |
| 32 | * can be checked directly) plus the `effect.exit` attribute the tracer |
| 33 | * forwards on span end. |
| 34 | */ |
| 35 | interface SpanRow extends workers.ObservabilitySharedQueriesGetResponseEventsEventsItemMetadata { |
| 36 | exit: unknown; |
| 37 | } |
| 38 | |
| 39 | const exitOf = (source: unknown): unknown => { |
| 40 | if (typeof source !== "object" || source === null) return undefined; |
| 41 | const effect = (source as Record<string, unknown>).effect; |
| 42 | return typeof effect === "object" && effect !== null |
| 43 | ? (effect as Record<string, unknown>).exit |
| 44 | : undefined; |
| 45 | }; |
| 46 | |
| 47 | const querySpans = (accountId: string, service: string) => |
| 48 | // The timeframe is re-evaluated per poll: rows are keyed by ingestion |
| 49 | // time, so a window fixed at the first attempt never sees late arrivals. |
| 50 | Effect.suspend(() => |
| 51 | workers.queryObservabilityTelemetry({ |
| 52 | accountId, |
| 53 | queryId: "adhoc-native-tracing", |
| 54 | view: "events", |
| 55 | timeframe: { from: Date.now() - 15 * 60 * 1000, to: Date.now() }, |
| 56 | limit: 500, |
| 57 | parameters: { |
| 58 | filters: [ |
| 59 | { |
| 60 | key: "$metadata.service", |
| 61 | operation: "eq", |
| 62 | type: "string", |
| 63 | value: service, |
| 64 | }, |
| 65 | { key: "$metadata.spanName", operation: "exists", type: "string" }, |
| 66 | ], |
| 67 | }, |
| 68 | }), |
| 69 | ).pipe( |
| 70 | Effect.map((response) => |
| 71 | (response.events?.events ?? []).map((event): SpanRow => ({ |
| 72 | ...event.metadata, |
| 73 | exit: exitOf(event.source), |
| 74 | })), |
| 75 | ), |
| 76 | ); |
| 77 | |
| 78 | /** The spans of one request: the root matched by name + `request.id`, and every row sharing its trace. */ |
| 79 | const traceOf = (spans: SpanRow[], root: string, id: string) => { |
| 80 | const rootSpan = spans.find((s) => s.spanName === root && s.requestId === id); |
| 81 | return rootSpan === undefined |
| 82 | ? undefined |
| 83 | : { |
| 84 | root: rootSpan, |
| 85 | spans: spans.filter((s) => s.traceId === rootSpan.traceId), |
| 86 | /** Exactly one span of this name in the trace. */ |
| 87 | span: (name: string) => { |
| 88 | const matches = spans.filter( |
| 89 | (s) => s.traceId === rootSpan.traceId && s.spanName === name, |
| 90 | ); |
| 91 | expect(matches).toHaveLength(1); |
| 92 | return matches[0]!; |
| 93 | }, |
| 94 | }; |
| 95 | }; |
| 96 | |
| 97 | type Bindings = ReadonlyArray<ResourceBinding<Worker["Binding"]>>; |
| 98 | |
| 99 | const tracesBind = (traces: NonNullable<Worker["Binding"]["observability"]>["traces"]): Bindings => |
| 100 | [{ sid: "Cloudflare.Telemetry", data: { observability: { traces } } }] as unknown as Bindings; |
| 101 | |
| 102 | describe( |
| 103 | "resolveObservability", |
| 104 | { |
| 105 | tags: ["unit", "provider:cloudflare", "provider:cloudflare:worker", "local"], |
| 106 | }, |
| 107 | () => { |
| 108 | unit("omitted props + bound traces keep default logs", () => { |
| 109 | const resolved = resolveObservability({}, tracesBind({ enabled: true })); |
| 110 | expect(resolved.enabled).toBe(true); |
| 111 | expect(resolved.logs?.enabled).toBe(true); |
| 112 | expect(resolved.logs?.invocationLogs).toBe(true); |
| 113 | expect(resolved.traces?.enabled).toBe(true); |
| 114 | }); |
| 115 | |
| 116 | unit("explicit traces.enabled false wins over the bind", () => { |
| 117 | const resolved = resolveObservability( |
| 118 | { observability: { enabled: true, traces: { enabled: false } } }, |
| 119 | tracesBind({ enabled: true, persist: true }), |
| 120 | ); |
| 121 | expect(resolved.traces?.enabled).toBe(false); |
| 122 | expect(resolved.traces?.persist).toBeUndefined(); |
| 123 | }); |
| 124 | |
| 125 | unit("fills traces when news.observability exists without traces", () => { |
| 126 | const resolved = resolveObservability( |
| 127 | { |
| 128 | observability: { |
| 129 | enabled: true, |
| 130 | logs: { enabled: true, invocationLogs: true, persist: true }, |
| 131 | }, |
| 132 | }, |
| 133 | tracesBind({ enabled: true, headSamplingRate: 1 }), |
| 134 | ); |
| 135 | expect(resolved.logs?.persist).toBe(true); |
| 136 | expect(resolved.traces?.enabled).toBe(true); |
| 137 | expect(resolved.traces?.headSamplingRate).toBe(1); |
| 138 | }); |
| 139 | |
| 140 | unit("Issues coexist with Telemetry traces and explicit logs", () => { |
| 141 | const resolved = resolveObservability( |
| 142 | { |
| 143 | observability: { |
| 144 | enabled: true, |
| 145 | logs: { enabled: true, invocationLogs: true }, |
| 146 | issues: { enabled: true }, |
| 147 | }, |
| 148 | }, |
| 149 | tracesBind({ enabled: true, headSamplingRate: 0.1 }), |
| 150 | ); |
| 151 | expect(resolved.issues?.enabled).toBe(true); |
| 152 | expect(resolved.logs?.invocationLogs).toBe(true); |
| 153 | expect(resolved.traces?.enabled).toBe(true); |
| 154 | expect(resolved.traces?.headSamplingRate).toBe(0.1); |
| 155 | }); |
| 156 | |
| 157 | unit("no bind returns default logs", () => { |
| 158 | const resolved = resolveObservability({}, []); |
| 159 | expect(resolved.traces).toBeUndefined(); |
| 160 | expect(resolved.logs?.invocationLogs).toBe(true); |
| 161 | }); |
| 162 | }, |
| 163 | ); |
| 164 | |
| 165 | test.provider( |
| 166 | "Cloudflare.Telemetry() enables traces without clobbering default logs", |
| 167 | (stack) => |
| 168 | Effect.gen(function* () { |
| 169 | const { accountId } = yield* yield* CloudflareEnvironment; |
| 170 | yield* stack.destroy(); |
| 171 | |
| 172 | const worker = yield* stack.deploy( |
| 173 | Effect.gen(function* () { |
| 174 | return yield* NativeTracingWorker; |
| 175 | }), |
| 176 | ); |
| 177 | |
| 178 | const settings = yield* workers.getScriptScriptAndVersionSetting({ |
| 179 | accountId, |
| 180 | scriptName: worker.workerName, |
| 181 | }); |
| 182 | expect(settings.observability?.enabled).toBe(true); |
| 183 | expect(settings.observability?.logs?.enabled).toBe(true); |
| 184 | expect(settings.observability?.logs?.invocationLogs).toBe(true); |
| 185 | expect(settings.observability?.traces?.enabled).toBe(true); |
| 186 | |
| 187 | yield* expectUrlContains(`${worker.url}/work`, "native-did-work", { |
| 188 | timeout: "180 seconds", |
| 189 | }); |
| 190 | |
| 191 | yield* stack.destroy(); |
| 192 | yield* waitForWorkerToBeDeleted(worker.workerName, accountId); |
| 193 | }).pipe(logLevel), |
| 194 | { |
| 195 | tags: [ |
| 196 | "provider:cloudflare", |
| 197 | "provider:cloudflare:kv", |
| 198 | "provider:cloudflare:queue", |
| 199 | "provider:cloudflare:worker", |
| 200 | "live", |
| 201 | ], |
| 202 | timeout: 180_000, |
| 203 | }, |
| 204 | ); |
| 205 | |
| 206 | test.provider( |
| 207 | "explicit observability.traces wins over Cloudflare.Telemetry() bind", |
| 208 | (stack) => |
| 209 | Effect.gen(function* () { |
| 210 | const { accountId } = yield* yield* CloudflareEnvironment; |
| 211 | yield* stack.destroy(); |
| 212 | |
| 213 | const worker = yield* stack.deploy( |
| 214 | Effect.gen(function* () { |
| 215 | return yield* makeTracedWorker("NativeTracingOverride", { |
| 216 | observability: { |
| 217 | enabled: true, |
| 218 | traces: { enabled: false }, |
| 219 | }, |
| 220 | }); |
| 221 | }), |
| 222 | ); |
| 223 | |
| 224 | const settings = yield* workers.getScriptScriptAndVersionSetting({ |
| 225 | accountId, |
| 226 | scriptName: worker.workerName, |
| 227 | }); |
| 228 | expect(settings.observability?.traces?.enabled).toBe(false); |
| 229 | expect(settings.observability?.logs?.invocationLogs).toBe(true); |
| 230 | |
| 231 | yield* stack.destroy(); |
| 232 | yield* waitForWorkerToBeDeleted(worker.workerName, accountId); |
| 233 | }).pipe(logLevel), |
| 234 | { |
| 235 | tags: [ |
| 236 | "provider:cloudflare", |
| 237 | "provider:cloudflare:kv", |
| 238 | "provider:cloudflare:queue", |
| 239 | "provider:cloudflare:worker", |
| 240 | "live", |
| 241 | ], |
| 242 | timeout: 180_000, |
| 243 | }, |
| 244 | ); |
| 245 | |
| 246 | test.provider( |
| 247 | "Cloudflare.Telemetry() fails deploy on a pre-startActiveSpan compatibility date", |
| 248 | (stack) => |
| 249 | Effect.gen(function* () { |
| 250 | yield* stack.destroy(); |
| 251 | |
| 252 | const error = yield* stack |
| 253 | .deploy( |
| 254 | Effect.gen(function* () { |
| 255 | return yield* makeTracedWorker("NativeTracingOldDate", { |
| 256 | compatibility: { date: "2026-03-17" }, |
| 257 | }); |
| 258 | }), |
| 259 | ) |
| 260 | .pipe(Effect.flip); |
| 261 | |
| 262 | expect(error).toBeInstanceOf(CloudflareTelemetryCompatibilityError); |
| 263 | expect(String(error)).toContain(MIN_CLOUDFLARE_TRACING_DATE); |
| 264 | expect(String(error)).toContain("2026-03-17"); |
| 265 | |
| 266 | yield* stack.destroy(); |
| 267 | }).pipe(logLevel), |
| 268 | { |
| 269 | tags: [ |
| 270 | "provider:cloudflare", |
| 271 | "provider:cloudflare:kv", |
| 272 | "provider:cloudflare:queue", |
| 273 | "provider:cloudflare:worker", |
| 274 | "live", |
| 275 | ], |
| 276 | timeout: 120_000, |
| 277 | }, |
| 278 | ); |
| 279 | |
| 280 | test.provider( |
| 281 | "Effect spans appear in Workers Observability after Cloudflare.Telemetry()", |
| 282 | (stack) => |
| 283 | Effect.gen(function* () { |
| 284 | const { accountId } = yield* yield* CloudflareEnvironment; |
| 285 | yield* stack.destroy(); |
| 286 | |
| 287 | const worker = yield* stack.deploy( |
| 288 | Effect.gen(function* () { |
| 289 | return yield* NativeTracingWorker; |
| 290 | }), |
| 291 | ); |
| 292 | |
| 293 | yield* expectUrlContains(`${worker.url}/work`, "native-did-work", { |
| 294 | timeout: "180 seconds", |
| 295 | }); |
| 296 | |
| 297 | // Poll until the request's Effect spans have been ingested. |
| 298 | const spans = yield* querySpans(accountId, worker.workerName).pipe( |
| 299 | Effect.repeat({ |
| 300 | schedule: Schedule.spaced("5 seconds"), |
| 301 | until: (spans) => |
| 302 | spans.some((s) => s.spanName === "operation") && |
| 303 | spans.some((s) => s.spanName === "native.child"), |
| 304 | times: 36, |
| 305 | }), |
| 306 | ); |
| 307 | expect(spans.map((s) => s.spanName)).toEqual( |
| 308 | expect.arrayContaining(["operation", "native.child"]), |
| 309 | ); |
| 310 | |
| 311 | yield* stack.destroy(); |
| 312 | yield* waitForWorkerToBeDeleted(worker.workerName, accountId); |
| 313 | }).pipe(logLevel), |
| 314 | { |
| 315 | tags: [ |
| 316 | "provider:cloudflare", |
| 317 | "provider:cloudflare:kv", |
| 318 | "provider:cloudflare:queue", |
| 319 | "provider:cloudflare:worker", |
| 320 | "live", |
| 321 | ], |
| 322 | timeout: 300_000, |
| 323 | }, |
| 324 | ); |
| 325 | |
| 326 | const FANOUT_SPANS = [ |
| 327 | "operation", |
| 328 | "child.a", |
| 329 | "child.a.inner", |
| 330 | "child.b", |
| 331 | "child.b.inner", |
| 332 | "forked", |
| 333 | "forked.inner", |
| 334 | ]; |
| 335 | |
| 336 | test.provider( |
| 337 | "records the Effect span tree across fibers, exits, Durable Object and queue handlers", |
| 338 | (stack) => |
| 339 | Effect.gen(function* () { |
| 340 | const { accountId } = yield* yield* CloudflareEnvironment; |
| 341 | yield* stack.destroy(); |
| 342 | |
| 343 | const worker = yield* stack.deploy( |
| 344 | Effect.gen(function* () { |
| 345 | return yield* NativeTracingWorker; |
| 346 | }), |
| 347 | ); |
| 348 | yield* expectUrlContains(`${worker.url}/fanout?id=warmup`, "native-did-fanout", { |
| 349 | timeout: "180 seconds", |
| 350 | }); |
| 351 | |
| 352 | // Each request retries through workers.dev propagation (a colo can |
| 353 | // still serve Cloudflare's error page right after the warm-up |
| 354 | // succeeded elsewhere); a retried id just produces one more |
| 355 | // independent trace. |
| 356 | const hit = (path: string, id: string, marker = id) => |
| 357 | expectUrlContains(`${worker.url}${path}?id=${id}`, marker, { |
| 358 | timeout: "120 seconds", |
| 359 | }); |
| 360 | // One request shape per event path. A request's trace is located by |
| 361 | // the `request.id` annotated on its root span (`$metadata.requestId`) |
| 362 | // and is complete once every expected span arrived, plus a platform |
| 363 | // span (e.g. the KV read) under each span in `platformUnder`. |
| 364 | const shapes = [ |
| 365 | { |
| 366 | name: "fanout", |
| 367 | root: "operation", |
| 368 | names: FANOUT_SPANS, |
| 369 | platformUnder: ["child.a", "child.b", "forked"], |
| 370 | // Several in flight at once so fibers from different requests |
| 371 | // interleave inside the isolate. |
| 372 | send: (id: string) => hit("/fanout", id), |
| 373 | concurrent: 3, |
| 374 | }, |
| 375 | { |
| 376 | name: "exits", |
| 377 | root: "operation", |
| 378 | names: ["operation", "failing.child", "interrupted.child"], |
| 379 | platformUnder: [], |
| 380 | send: (id: string) => hit("/exits", id), |
| 381 | concurrent: 1, |
| 382 | }, |
| 383 | { |
| 384 | name: "rpc", |
| 385 | root: "do.operation", |
| 386 | names: ["do.operation", "do.inner"], |
| 387 | platformUnder: [], |
| 388 | send: (id: string) => hit("/rpc", id, "native-did-rpc:do-ok"), |
| 389 | concurrent: 1, |
| 390 | }, |
| 391 | { |
| 392 | name: "queue", |
| 393 | root: "queue.operation", |
| 394 | names: ["queue.operation", "queue.inner"], |
| 395 | platformUnder: [], |
| 396 | send: (id: string) => hit("/enqueue", id), |
| 397 | concurrent: 1, |
| 398 | }, |
| 399 | ] as const; |
| 400 | type Shape = (typeof shapes)[number]; |
| 401 | const complete = (spans: SpanRow[], shape: Shape, id: string) => { |
| 402 | const trace = traceOf(spans, shape.root, id); |
| 403 | return ( |
| 404 | trace !== undefined && |
| 405 | shape.names.every((name) => trace.spans.some((s) => s.spanName === name)) && |
| 406 | shape.platformUnder.every((name) => |
| 407 | trace.spans.some( |
| 408 | (parent) => |
| 409 | parent.spanName === name && |
| 410 | trace.spans.some( |
| 411 | (s) => |
| 412 | s.parentSpanId === parent.spanId && !FANOUT_SPANS.includes(s.spanName ?? ""), |
| 413 | ), |
| 414 | ), |
| 415 | ) |
| 416 | ); |
| 417 | }; |
| 418 | // Every id sent per shape, so a trace that arrives late still counts. |
| 419 | const sent = new Map<Shape, string[]>(shapes.map((shape) => [shape, []])); |
| 420 | const send = (shape: Shape) => |
| 421 | Effect.gen(function* () { |
| 422 | const ids = yield* Effect.sync(() => { |
| 423 | const ids = Array.from({ length: shape.concurrent }, () => |
| 424 | globalThis.crypto.randomUUID(), |
| 425 | ); |
| 426 | sent.get(shape)!.push(...ids); |
| 427 | return ids; |
| 428 | }); |
| 429 | yield* Effect.all(ids.map(shape.send), { concurrency: "unbounded" }); |
| 430 | }); |
| 431 | yield* Effect.all( |
| 432 | [ |
| 433 | ...shapes.map(send), |
| 434 | // Head sampling 1: every Effect span is sampled. |
| 435 | expectUrlContains(`${worker.url}/sampled`, '"operation":true,"child":true', { |
| 436 | timeout: "120 seconds", |
| 437 | }), |
| 438 | ], |
| 439 | { concurrency: "unbounded" }, |
| 440 | ); |
| 441 | |
| 442 | // Ingestion latency varies from seconds to several minutes, and |
| 443 | // ingestion is lossy: a span and the platform spans under it can go |
| 444 | // missing for good, so waiting on one request can never finish. Every |
| 445 | // minute, send a fresh request for each shape that has no complete |
| 446 | // trace yet; the assertions below run on the traces that arrived whole. |
| 447 | const completeIds = (spans: SpanRow[], shape: Shape) => |
| 448 | sent.get(shape)!.filter((id) => complete(spans, shape, id)); |
| 449 | let polls = 0; |
| 450 | const poll = Effect.gen(function* () { |
| 451 | const spans = yield* querySpans(accountId, worker.workerName); |
| 452 | const missing = shapes.filter((shape) => completeIds(spans, shape).length === 0); |
| 453 | const count = yield* Effect.sync(() => ++polls); |
| 454 | if (missing.length > 0 && count % 12 === 0) { |
| 455 | yield* Effect.logInfo( |
| 456 | `native-tracing: no complete trace yet for ${missing.map((s) => s.name).join(", ")}; sending again`, |
| 457 | ); |
| 458 | yield* Effect.all(missing.map(send), { concurrency: "unbounded" }); |
| 459 | } |
| 460 | return { spans, missing }; |
| 461 | }); |
| 462 | const { spans, missing } = yield* poll.pipe( |
| 463 | Effect.repeat({ |
| 464 | schedule: Schedule.spaced("5 seconds"), |
| 465 | until: (result) => result.missing.length === 0, |
| 466 | times: 60, |
| 467 | }), |
| 468 | ); |
| 469 | if (missing.length > 0) { |
| 470 | yield* Effect.logError( |
| 471 | `native-tracing: no complete trace for ${missing.map((s) => s.name).join(", ")}; spans ingested so far: ${JSON.stringify( |
| 472 | spans.map((s) => [s.requestId, s.traceId, s.spanName]), |
| 473 | )}`, |
| 474 | ); |
| 475 | } |
| 476 | expect(missing.map((shape) => shape.name)).toEqual([]); |
| 477 | const [fanout, exits, rpc, queue] = shapes; |
| 478 | const ids = { |
| 479 | fanout: completeIds(spans, fanout), |
| 480 | exits: completeIds(spans, exits)[0]!, |
| 481 | rpc: completeIds(spans, rpc)[0]!, |
| 482 | queue: completeIds(spans, queue)[0]!, |
| 483 | }; |
| 484 | |
| 485 | // Fan-out: siblings and nesting follow the Effect span tree, and each |
| 486 | // KV read (a Cloudflare auto-instrumented span) is attributed to the |
| 487 | // branch whose fiber issued it — even though the scheduler resumed |
| 488 | // that fiber on a timer tick outside `operation`'s async context. |
| 489 | for (const id of ids.fanout) { |
| 490 | const trace = traceOf(spans, "operation", id)!; |
| 491 | const operation = trace.root; |
| 492 | // `operation` hangs off Cloudflare's own request span. |
| 493 | expect( |
| 494 | trace.spans.some( |
| 495 | (s) => s.spanId === operation.parentSpanId && s.spanName !== "operation", |
| 496 | ), |
| 497 | ).toBe(true); |
| 498 | for (const name of ["child.a", "child.b", "forked"]) { |
| 499 | const branch = trace.span(name); |
| 500 | expect(branch.parentSpanId).toBe(operation.spanId); |
| 501 | expect(trace.span(`${name}.inner`).parentSpanId).toBe(branch.spanId); |
| 502 | const platform = trace.spans.filter( |
| 503 | (s) => s.parentSpanId === branch.spanId && !FANOUT_SPANS.includes(s.spanName ?? ""), |
| 504 | ); |
| 505 | expect(platform.length).toBeGreaterThanOrEqual(1); |
| 506 | } |
| 507 | expect(operation.exit).toBe("success"); |
| 508 | } |
| 509 | |
| 510 | // Exits: the failing child (caught by its parent) and the interrupted |
| 511 | // child record their outcome as `effect.exit`. |
| 512 | { |
| 513 | const trace = traceOf(spans, "operation", ids.exits)!; |
| 514 | expect(trace.root.exit).toBe("success"); |
| 515 | const failing = trace.span("failing.child"); |
| 516 | expect(failing.parentSpanId).toBe(trace.root.spanId); |
| 517 | expect(failing.exit).toBe("failure"); |
| 518 | const interrupted = trace.span("interrupted.child"); |
| 519 | expect(interrupted.parentSpanId).toBe(trace.root.spanId); |
| 520 | expect(interrupted.exit).toBe("interrupted"); |
| 521 | } |
| 522 | |
| 523 | // Durable Object RPC: spans opened inside the DO method nest under |
| 524 | // the DO invocation's platform span. |
| 525 | { |
| 526 | const trace = traceOf(spans, "do.operation", ids.rpc)!; |
| 527 | expect(trace.root.parentSpanId).toBeTruthy(); |
| 528 | expect(trace.span("do.inner").parentSpanId).toBe(trace.root.spanId); |
| 529 | expect(trace.root.exit).toBe("success"); |
| 530 | } |
| 531 | |
| 532 | // Queue consumer: same for the `queue` event handler. |
| 533 | { |
| 534 | const trace = traceOf(spans, "queue.operation", ids.queue)!; |
| 535 | expect(trace.root.parentSpanId).toBeTruthy(); |
| 536 | expect(trace.span("queue.inner").parentSpanId).toBe(trace.root.spanId); |
| 537 | expect(trace.root.exit).toBe("success"); |
| 538 | } |
| 539 | |
| 540 | yield* stack.destroy(); |
| 541 | yield* waitForWorkerToBeDeleted(worker.workerName, accountId); |
| 542 | }).pipe(logLevel), |
| 543 | { |
| 544 | tags: [ |
| 545 | "provider:cloudflare", |
| 546 | "provider:cloudflare:kv", |
| 547 | "provider:cloudflare:queue", |
| 548 | "provider:cloudflare:worker", |
| 549 | "live", |
| 550 | ], |
| 551 | timeout: 480_000, |
| 552 | }, |
| 553 | ); |
| 554 | |
| 555 | // The OTLP collector the export test's observability destination pushes |
| 556 | // to. Cloudflare's exporter needs a public hostname, so the collector |
| 557 | // gets a deterministic custom domain on the standing test zone. |
| 558 | const zoneName = process.env.CLOUDFLARE_TEST_DNS_ZONE_NAME ?? "alchemy-test-2.us"; |
| 559 | const collectorHost = `native-trace-collector-${stageSuffix()}.${zoneName}`; |
| 560 | // Named up front: the traced Worker lists the destination by name. |
| 561 | const destinationName = `native-trace-export-${stageSuffix()}`; |
| 562 | const collectorMain = pathe.resolve(import.meta.dirname, "fixtures/native-tracing/collector.ts"); |
| 563 | |
| 564 | /** An exported OTLP span, reduced to what the status assertions read. */ |
| 565 | interface ExportedSpan { |
| 566 | traceId: string; |
| 567 | name: string; |
| 568 | requestId: unknown; |
| 569 | status: { code: "unset" | "ok" | "error"; message?: string }; |
| 570 | } |
| 571 | |
| 572 | // OTLP/JSON encodes the status code as its enum number or its name. |
| 573 | const statusCode = (code: unknown): ExportedSpan["status"]["code"] => |
| 574 | code === 2 || code === "STATUS_CODE_ERROR" |
| 575 | ? "error" |
| 576 | : code === 1 || code === "STATUS_CODE_OK" |
| 577 | ? "ok" |
| 578 | : "unset"; |
| 579 | |
| 580 | // The destination's preflight POST stores a non-OTLP body too. |
| 581 | const parseJson = (text: string): any => { |
| 582 | try { |
| 583 | return JSON.parse(text); |
| 584 | } catch { |
| 585 | return undefined; |
| 586 | } |
| 587 | }; |
| 588 | |
| 589 | const exportedSpans = (items: ReadonlyArray<string>): ExportedSpan[] => |
| 590 | items.flatMap((item) => { |
| 591 | const body = parseJson(item); |
| 592 | return (body?.resourceSpans ?? []).flatMap((resource: any) => |
| 593 | (resource.scopeSpans ?? []).flatMap((scope: any) => |
| 594 | (scope.spans ?? []).map((span: any): ExportedSpan => ({ |
| 595 | traceId: span.traceId, |
| 596 | name: span.name, |
| 597 | requestId: (span.attributes ?? []).find((a: any) => a.key === "request.id")?.value |
| 598 | ?.stringValue, |
| 599 | status: { code: statusCode(span.status?.code), message: span.status?.message }, |
| 600 | })), |
| 601 | ), |
| 602 | ); |
| 603 | }); |
| 604 | |
| 605 | test.provider( |
| 606 | "exports a failed Effect span with error status over OTLP", |
| 607 | (stack) => |
| 608 | Effect.gen(function* () { |
| 609 | const { accountId } = yield* yield* CloudflareEnvironment; |
| 610 | yield* stack.destroy(); |
| 611 | |
| 612 | // The traced Worker lists the destination by name, so the collector |
| 613 | // and destination deploy first and stay declared in the second step. |
| 614 | const sink = Effect.gen(function* () { |
| 615 | const collector = yield* Cloudflare.Worker("NativeTraceCollector", { |
| 616 | main: collectorMain, |
| 617 | env: { SINK: Cloudflare.DurableObject<TraceSink>("TraceSink") }, |
| 618 | domain: collectorHost, |
| 619 | compatibility: { date: "2024-09-23" }, |
| 620 | }); |
| 621 | yield* Cloudflare.Workers.ObservabilityDestination("NativeTraceDestination", { |
| 622 | name: destinationName, |
| 623 | url: `https://${collectorHost}/v1/traces`, |
| 624 | logpushDataset: "opentelemetry-traces", |
| 625 | // The collector's custom domain may not serve yet on create. |
| 626 | skipPreflightCheck: true, |
| 627 | }); |
| 628 | return collector; |
| 629 | }); |
| 630 | yield* stack.deploy(sink); |
| 631 | const { collector, worker } = yield* stack.deploy( |
| 632 | Effect.gen(function* () { |
| 633 | const collector = yield* sink; |
| 634 | const worker = yield* makeTracedWorker("NativeTracingExport", { |
| 635 | observability: { |
| 636 | enabled: true, |
| 637 | traces: { enabled: true, destinations: [destinationName] }, |
| 638 | }, |
| 639 | }); |
| 640 | return { collector, worker }; |
| 641 | }), |
| 642 | ); |
| 643 | |
| 644 | // Read back over workers.dev: a custom domain attached makes |
| 645 | // `collector.url` the domain, so derive the workers.dev URL. |
| 646 | const { subdomain } = yield* workers.getSubdomain({ accountId }); |
| 647 | const collected = `https://${collector.workerName}.${subdomain}.workers.dev/collected`; |
| 648 | yield* expectUrlContains(collected, "trace-collector-ok", { timeout: "180 seconds" }); |
| 649 | yield* expectUrlContains(`${worker.url}/exits?id=warmup`, "native-did-exits", { |
| 650 | timeout: "180 seconds", |
| 651 | }); |
| 652 | |
| 653 | // Export runs through Logpush batches, so a request's spans can take |
| 654 | // minutes to arrive. Send a fresh `/exits` request every minute until |
| 655 | // one request's whole trace has been exported. |
| 656 | const client = yield* HttpClient.HttpClient; |
| 657 | const sent: string[] = []; |
| 658 | const send = Effect.gen(function* () { |
| 659 | const id = yield* Effect.sync(() => { |
| 660 | const id = globalThis.crypto.randomUUID(); |
| 661 | sent.push(id); |
| 662 | return id; |
| 663 | }); |
| 664 | yield* expectUrlContains(`${worker.url}/exits?id=${id}`, id, { timeout: "120 seconds" }); |
| 665 | }); |
| 666 | const traceFor = (spans: ExportedSpan[]) => |
| 667 | sent |
| 668 | .map((id) => { |
| 669 | const root = spans.find((s) => s.name === "operation" && s.requestId === id); |
| 670 | const trace = spans.filter((s) => s.traceId === root?.traceId); |
| 671 | const span = (name: string) => trace.find((s) => s.name === name); |
| 672 | return root && span("failing.child") && span("interrupted.child") |
| 673 | ? { root, failing: span("failing.child")!, interrupted: span("interrupted.child")! } |
| 674 | : undefined; |
| 675 | }) |
| 676 | .find((trace) => trace !== undefined); |
| 677 | let polls = 0; |
| 678 | const trace = yield* Effect.gen(function* () { |
| 679 | const count = yield* Effect.sync(() => polls++); |
| 680 | if (count % 12 === 0) yield* send; |
| 681 | const body = (yield* client.get(collected).pipe(Effect.flatMap((res) => res.json))) as { |
| 682 | items: string[]; |
| 683 | }; |
| 684 | return traceFor(exportedSpans(body.items)); |
| 685 | }).pipe( |
| 686 | Effect.repeat({ |
| 687 | schedule: Schedule.spaced("5 seconds"), |
| 688 | until: (trace) => trace !== undefined, |
| 689 | times: 72, |
| 690 | }), |
| 691 | ); |
| 692 | |
| 693 | expect(trace).toBeDefined(); |
| 694 | // A failure maps to the standard error status; success and pure |
| 695 | // interruption stay unset. |
| 696 | expect(trace!.failing.status).toEqual({ code: "error", message: "boom" }); |
| 697 | expect(trace!.interrupted.status.code).toBe("unset"); |
| 698 | expect(trace!.root.status.code).toBe("unset"); |
| 699 | |
| 700 | yield* stack.destroy(); |
| 701 | yield* waitForWorkerToBeDeleted(worker.workerName, accountId); |
| 702 | }).pipe(logLevel), |
| 703 | { |
| 704 | tags: [ |
| 705 | "provider:cloudflare", |
| 706 | "provider:cloudflare:kv", |
| 707 | "provider:cloudflare:queue", |
| 708 | "provider:cloudflare:worker", |
| 709 | "live", |
| 710 | ], |
| 711 | timeout: 480_000, |
| 712 | }, |
| 713 | ); |
| 714 | |
| 715 | test.provider( |
| 716 | "head sampling below 1 is applied by Cloudflare at ingestion", |
| 717 | (stack) => |
| 718 | Effect.gen(function* () { |
| 719 | const { accountId } = yield* yield* CloudflareEnvironment; |
| 720 | const currentConfig = yield* ConfigProvider.ConfigProvider; |
| 721 | yield* stack.destroy(); |
| 722 | |
| 723 | const worker = yield* stack |
| 724 | .deploy( |
| 725 | Effect.gen(function* () { |
| 726 | return yield* makeTracedWorker("NativeTracingSampled"); |
| 727 | }), |
| 728 | ) |
| 729 | .pipe( |
| 730 | Effect.provideService( |
| 731 | ConfigProvider.ConfigProvider, |
| 732 | ConfigProvider.orElse( |
| 733 | ConfigProvider.fromUnknown({ HEAD_SAMPLING_RATE: 0.5 }), |
| 734 | currentConfig, |
| 735 | ), |
| 736 | ), |
| 737 | ); |
| 738 | |
| 739 | const settings = yield* workers.getScriptScriptAndVersionSetting({ |
| 740 | accountId, |
| 741 | scriptName: worker.workerName, |
| 742 | }); |
| 743 | expect(settings.observability?.traces?.enabled).toBe(true); |
| 744 | expect(settings.observability?.traces?.headSamplingRate).toBe(0.5); |
| 745 | |
| 746 | // The runtime reports every invocation as traced (`span.isTraced` does |
| 747 | // not carry the head-sampling decision — a rate of 0 behaves the same), |
| 748 | // so Effect's `sampled` flag stays true and consistent parent → child. |
| 749 | // Sampling is applied when Cloudflare ingests: of N invocations, only |
| 750 | // ~half persist. |
| 751 | // |
| 752 | // A 50% rate (rather than a few percent) keeps this test fast and |
| 753 | // two-sided: some request of the batch is always persisted, so the |
| 754 | // ingestion wait ends as soon as rows land instead of at its ceiling, |
| 755 | // and the count can be bounded from below as well as above. |
| 756 | const runId = yield* Effect.sync(() => globalThis.crypto.randomUUID()); |
| 757 | yield* expectUrlContains(`${worker.url}/sampled?id=warmup-${runId}`, "native-did-sample", { |
| 758 | timeout: "180 seconds", |
| 759 | }); |
| 760 | const client = yield* HttpClient.HttpClient; |
| 761 | const probe = client.get(`${worker.url}/sampled?id=${runId}`).pipe( |
| 762 | Effect.flatMap((res) => res.json), |
| 763 | Effect.map((body) => body as { operation: boolean; child: boolean }), |
| 764 | Effect.retry({ schedule: Schedule.spaced("1 second"), times: 30 }), |
| 765 | ); |
| 766 | const requests = 100; |
| 767 | const seen = yield* Effect.all( |
| 768 | Array.from({ length: requests }, () => probe), |
| 769 | { concurrency: 10 }, |
| 770 | ); |
| 771 | expect(seen.filter((b) => b.child !== b.operation)).toEqual([]); |
| 772 | |
| 773 | // One `operation` root per persisted request, keyed by this run's id |
| 774 | // so earlier runs inside the 15-minute query window don't count. |
| 775 | // Rows land in batches, so wait until two consecutive reads agree |
| 776 | // (and enough exist to prove tracing is on) before bounding the count. |
| 777 | // Binomial(100, 0.5) is within [20, 80] with P(miss) ≈ 1e-9; a few |
| 778 | // retried probes add at most a handful of extra traces. |
| 779 | const persistedCount = querySpans(accountId, worker.workerName).pipe( |
| 780 | Effect.map( |
| 781 | (spans) => |
| 782 | spans.filter((s) => s.spanName === "operation" && s.requestId === runId).length, |
| 783 | ), |
| 784 | ); |
| 785 | let previous = -1; |
| 786 | const persisted = yield* persistedCount.pipe( |
| 787 | Effect.repeat({ |
| 788 | schedule: Schedule.spaced("5 seconds"), |
| 789 | until: (count) => { |
| 790 | const settled = count >= 20 && count === previous; |
| 791 | previous = count; |
| 792 | return settled; |
| 793 | }, |
| 794 | times: 36, |
| 795 | }), |
| 796 | ); |
| 797 | expect(persisted).toBeGreaterThanOrEqual(20); |
| 798 | expect(persisted).toBeLessThanOrEqual(80); |
| 799 | |
| 800 | yield* stack.destroy(); |
| 801 | yield* waitForWorkerToBeDeleted(worker.workerName, accountId); |
| 802 | }).pipe(logLevel), |
| 803 | { |
| 804 | tags: [ |
| 805 | "provider:cloudflare", |
| 806 | "provider:cloudflare:kv", |
| 807 | "provider:cloudflare:queue", |
| 808 | "provider:cloudflare:worker", |
| 809 | "live", |
| 810 | ], |
| 811 | timeout: 300_000, |
| 812 | }, |
| 813 | ); |