The best way to control Claude Code, Codex and more from any computer or phone

main
ts 639 lines 21.5 KB
Raw
1 import * as logpush from "@distilled.cloud/cloudflare/logpush";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { createPhysicalName } from "alchemy/PhysicalName";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import * as Data from "effect/Data";
7 import * as Effect from "effect/Effect";
8 import * as Predicate from "effect/Predicate";
9 import * as Stream from "effect/Stream";
10 import { CloudflareEnvironment } from "../CloudflareEnvironment.ts";
11 import type { Providers } from "../Providers.ts";
12 import { listAllZones } from "../Zone/lookup.ts";
13
14 export class JobUnexpectedResponse extends Data.TaggedError("JobUnexpectedResponse")<{
15 readonly message: string;
16 readonly cause?: unknown;
17 }> {}
18
19 const TypeId = "Cloudflare.Logpush.Job" as const;
20 type TypeId = typeof TypeId;
21
22 /**
23 * Name of the dataset a Logpush job pushes. The available datasets depend
24 * on the job's scope (account vs zone) and the account's plan — e.g.
25 * `workers_trace_events` and `audit_logs` are account-scoped, while
26 * `http_requests` and `firewall_events` are zone-scoped (Enterprise).
27 */
28 export type Dataset = Exclude<NonNullable<logpush.CreateJobForAccountRequest["dataset"]>, null>;
29
30 /**
31 * Structured output configuration for a Logpush job — the replacement for
32 * the deprecated `logpull_options` string.
33 */
34 export interface OutputOptions {
35 /**
36 * String prepended to each batch (e.g. `[` to build a JSON array).
37 */
38 batchPrefix?: string;
39 /**
40 * String appended to each batch (e.g. `]`).
41 */
42 batchSuffix?: string;
43 /**
44 * Mitigation flag for CVE-2021-44228 — when `true`, verbatim
45 * `${` sequences are replaced with `x{`.
46 */
47 "cve-2021-44228"?: boolean;
48 /**
49 * Delimiter between fields in `csv` output.
50 */
51 fieldDelimiter?: string;
52 /**
53 * Log fields to include in the output. Available fields per dataset are
54 * listed on the Cloudflare developer docs.
55 */
56 fieldNames?: string[];
57 /**
58 * Whether to merge sub-request data into the parent request record
59 * (only meaningful for some datasets).
60 */
61 mergeSubrequests?: boolean;
62 /**
63 * Output format.
64 * @default "ndjson"
65 */
66 outputType?: "ndjson" | "csv";
67 /**
68 * Delimiter inserted between records.
69 */
70 recordDelimiter?: string;
71 /**
72 * String prepended to each record.
73 */
74 recordPrefix?: string;
75 /**
76 * String appended to each record.
77 */
78 recordSuffix?: string;
79 /**
80 * Go-template string used to render each record (mutually exclusive
81 * with prefix/suffix/delimiter options).
82 */
83 recordTemplate?: string;
84 /**
85 * Floating point fraction (0.0–1.0) of records to include.
86 * @default 1
87 */
88 sampleRate?: number;
89 /**
90 * Timestamp rendering format.
91 * @default "unixnano"
92 */
93 timestampFormat?: "unixnano" | "unix" | "rfc3339" | "rfc3339ms" | "rfc3339ns";
94 }
95
96 export interface JobProps {
97 /**
98 * Zone the job is scoped to. When omitted, the job is account-scoped
99 * (using the account from the active Cloudflare credentials).
100 *
101 * Stable — moving a job between scopes triggers a replacement.
102 */
103 zoneId?: string;
104 /**
105 * Name of the dataset to push (e.g. `workers_trace_events`,
106 * `audit_logs`, `http_requests`).
107 *
108 * Stable — the dataset is fixed at creation, so changing it triggers
109 * a replacement.
110 */
111 dataset: Dataset;
112 /**
113 * Destination URI, including any credentials the destination needs —
114 * e.g. `r2://bucket/{DATE}?account-id=…&access-key-id=…&secret-access-key=…`
115 * for R2, or an `s3://`, `gs://`, `https://` endpoint.
116 *
117 * Cloudflare validates the destination synchronously on create/update by
118 * writing a test object, so the destination (e.g. the R2 bucket) must
119 * already exist.
120 *
121 * Mutable — but Cloudflare may reject changing the destination
122 * *provider/domain* of an existing job; in that case change a
123 * replacement-triggering prop instead.
124 */
125 destinationConf: string;
126 /**
127 * Optional human readable job name (not unique). If omitted, a unique
128 * name will be generated.
129 * @default ${app}-${stage}-${id}
130 */
131 name?: string;
132 /**
133 * Whether the job actively pushes logs.
134 * @default false
135 */
136 enabled?: boolean;
137 /**
138 * Filter expression (JSON-encoded) selecting which events to push.
139 * See https://developers.cloudflare.com/logs/reference/filters/.
140 */
141 filter?: string;
142 /**
143 * Differentiates Logpush (`""`) from Edge Log Delivery (`"edge"`) jobs.
144 *
145 * Stable — changing the kind triggers a replacement.
146 * @default ""
147 */
148 kind?: "" | "edge";
149 /**
150 * Structured output configuration (fields, format, delimiters, sampling).
151 */
152 outputOptions?: OutputOptions;
153 /**
154 * Maximum uncompressed file size of a batch, in bytes. Between 5 MB and
155 * 1 GB, or `0` to disable the limit.
156 * @default 0
157 */
158 maxUploadBytes?: number;
159 /**
160 * Maximum interval in seconds between log batches. Between 30 and 300,
161 * or `0` to disable the limit.
162 * @default 30
163 */
164 maxUploadIntervalSeconds?: number;
165 /**
166 * Maximum number of log lines per batch. Between 1,000 and 1,000,000,
167 * or `0` to disable the limit.
168 * @default 100000
169 */
170 maxUploadRecords?: number;
171 /**
172 * Ownership challenge token proving destination ownership — required
173 * for destinations like S3/GCS/Azure. R2 destinations authenticated via
174 * credentials in `destinationConf` do not need it.
175 *
176 * Write-only: Cloudflare never echoes it back, so it does not
177 * participate in drift detection.
178 */
179 ownershipChallenge?: string;
180 }
181
182 export interface JobAttributes {
183 /** Cloudflare-assigned numeric job id. */
184 jobId: number;
185 /** Account that owns the job. */
186 accountId: string;
187 /** Zone the job is scoped to, or `undefined` for account-scoped jobs. */
188 zoneId: string | undefined;
189 /** Job name. */
190 name: string;
191 /** Dataset the job pushes. */
192 dataset: Dataset;
193 /**
194 * Destination URI. Note Cloudflare redacts embedded secrets (e.g. the
195 * R2 `secret-access-key`) when echoing this back.
196 */
197 destinationConf: string;
198 /** Whether the job is enabled. */
199 enabled: boolean;
200 /** Job kind (`""` for Logpush, `"edge"` for Edge Log Delivery). */
201 kind: string;
202 /** Last failure message, if the job is currently failing. */
203 errorMessage: string | undefined;
204 /** End of the last successfully-pushed log range. */
205 lastComplete: string | undefined;
206 /** Time of the last push failure. */
207 lastError: string | undefined;
208 }
209
210 export type Job = Resource<TypeId, JobProps, JobAttributes, never, Providers>;
211
212 /**
213 * A Cloudflare Logpush job that pushes batches of logs (HTTP requests,
214 * Workers trace events, audit logs, Zero Trust datasets, …) to a
215 * destination such as R2, S3, GCS, or an HTTP endpoint.
216 *
217 * Jobs can be account-scoped (default) or zone-scoped (pass `zoneId`).
218 * The dataset, kind, and scope are fixed at creation — changing any of
219 * them triggers a replacement; everything else updates in place.
220 * ### Pushing Workers trace events to R2
221 * **Example:** Account-scoped job writing to an R2 bucket
222 * The R2 destination authenticates with S3-compatible credentials embedded
223 * in the destination URI, so no ownership challenge is required.
224 * ```typescript
225 * const bucket = yield* Cloudflare.R2.Bucket("logs", {});
226 *
227 * const job = yield* Cloudflare.Logpush.Job("worker-logs", {
228 * dataset: "workers_trace_events",
229 * destinationConf: Output.interpolate`r2://${bucket.bucketName}/{DATE}?account-id=${accountId}&access-key-id=${r2AccessKeyId}&secret-access-key=${r2SecretAccessKey}`,
230 * enabled: true,
231 * });
232 * ```
233 *
234 * ### Zone-scoped jobs
235 * **Example:** HTTP requests dataset on a zone (Enterprise)
236 * ```typescript
237 * const job = yield* Cloudflare.Logpush.Job("http-logs", {
238 * zoneId: zone.zoneId,
239 * dataset: "http_requests",
240 * destinationConf: "s3://my-bucket/logs?region=us-east-1",
241 * ownershipChallenge: "00000000000000000000",
242 * });
243 * ```
244 *
245 * ### Output configuration
246 * **Example:** Selecting fields and batching limits
247 * ```typescript
248 * const job = yield* Cloudflare.Logpush.Job("worker-logs", {
249 * dataset: "workers_trace_events",
250 * destinationConf,
251 * outputOptions: {
252 * fieldNames: ["EventTimestampMs", "Outcome", "ScriptName"],
253 * outputType: "ndjson",
254 * timestampFormat: "rfc3339",
255 * },
256 * maxUploadIntervalSeconds: 60,
257 * });
258 * ```
259 *
260 * @see https://developers.cloudflare.com/logs/logpush/
261 *
262 * @resource
263 * @product Logpush
264 * @category Observability & Analytics
265 */
266 export const Job = Resource<Job>(TypeId);
267
268 /**
269 * Returns true if the given value is a Job resource.
270 */
271 export const isJob = (value: unknown): value is Job =>
272 Predicate.hasProperty(value, "Type") && value.Type === TypeId;
273
274 export const JobProvider = () =>
275 Provider.succeed(Job, {
276 stables: ["jobId", "accountId", "zoneId", "dataset", "kind"],
277
278 diff: Effect.fn(function* ({ olds = {}, news, output }) {
279 const { accountId } = yield* yield* CloudflareEnvironment;
280 const o = olds as JobProps;
281 const n = news as JobProps;
282 if ((output?.accountId ?? accountId) !== accountId) {
283 return { action: "replace" } as const;
284 }
285 if (o.dataset !== undefined && o.dataset !== n.dataset) {
286 return { action: "replace" } as const;
287 }
288 if (o.dataset !== undefined && (o.kind ?? "") !== (n.kind ?? "")) {
289 return { action: "replace" } as const;
290 }
291 // zoneId is string; only comparable when both sides are
292 // concrete strings (or one side is absent => scope change).
293 const oZone = typeof o.zoneId === "string" ? o.zoneId : undefined;
294 const nZone = typeof n.zoneId === "string" ? n.zoneId : undefined;
295 if (
296 o.dataset !== undefined &&
297 (typeof o.zoneId === "string" || o.zoneId === undefined) &&
298 (typeof n.zoneId === "string" || n.zoneId === undefined) &&
299 oZone !== nZone
300 ) {
301 return { action: "replace" } as const;
302 }
303 }),
304
305 read: Effect.fn(function* ({ id, output, olds }) {
306 const { accountId } = yield* yield* CloudflareEnvironment;
307 const scope: Scope = {
308 accountId: output?.accountId ?? accountId,
309 zoneId: output?.zoneId ?? (olds?.zoneId as string | undefined),
310 };
311 // Owned path: refresh by the persisted job id.
312 if (output?.jobId !== undefined) {
313 const observed = yield* observeById(scope, output.jobId);
314 const attrs = toAttributes(observed, scope);
315 if (attrs) return attrs;
316 }
317 // Cold/adoption path: no persisted id — find the job by name. The
318 // engine-generated physical name embeds the instance id, so a match
319 // on a generated name is proof we created it. A match on a
320 // user-provided name is not — report it `Unowned` so the engine
321 // gates takeover behind the adopt policy.
322 const name = yield* createJobName(id, olds?.name);
323 const match = yield* findByName(scope, name, olds?.dataset);
324 const attrs = toAttributes(match, scope);
325 if (attrs) {
326 return olds?.name !== undefined ? Unowned(attrs) : attrs;
327 }
328 return undefined;
329 }),
330
331 // Logpush jobs are hybrid-scoped: account-scoped jobs live under the
332 // account, zone-scoped jobs under each zone. Enumerate both — the account
333 // collection plus a fan-out over every zone — and hydrate each into the
334 // same Attributes shape `read` returns.
335 list: Effect.fn(function* () {
336 const { accountId } = yield* yield* CloudflareEnvironment;
337
338 const accountRows = yield* logpush.listJobsForAccount.pages({ accountId }).pipe(
339 Stream.runCollect,
340 Effect.map((chunk) =>
341 Array.from(chunk).flatMap((page) =>
342 (page.result ?? [])
343 .filter((job): job is NonNullable<typeof job> => job != null)
344 .map((job) => toAttributes(job, { accountId, zoneId: undefined }))
345 .filter((attrs): attrs is JobAttributes => attrs !== undefined),
346 ),
347 ),
348 );
349
350 const zones = yield* listAllZones(accountId);
351 const zoneRows = yield* Effect.forEach(
352 zones,
353 (zone) =>
354 logpush.listJobsForZone.pages({ zoneId: zone.id }).pipe(
355 Stream.runCollect,
356 Effect.map((chunk) =>
357 Array.from(chunk).flatMap((page) =>
358 (page.result ?? [])
359 .filter((job): job is NonNullable<typeof job> => job != null)
360 .map((job) => toAttributes(job, { accountId, zoneId: zone.id }))
361 .filter((attrs): attrs is JobAttributes => attrs !== undefined),
362 ),
363 ),
364 // Best-effort account-wide fan-out: a zone the token can't read
365 // for Logpush (plan-gated route, missing permission, or a code-
366 // 10000 auth blip) must be skipped, not fail the whole
367 // enumeration. Drop only that zone and keep the rest.
368 Effect.catchTag(["InvalidRoute", "Unauthorized", "Forbidden", "NotFound"], () =>
369 Effect.succeed([]),
370 ),
371 ),
372 { concurrency: 10 },
373 );
374
375 return [...accountRows, ...zoneRows.flat()];
376 }),
377
378 reconcile: Effect.fn(function* ({ id, news, olds, output }) {
379 const { accountId } = yield* yield* CloudflareEnvironment;
380 const scope: Scope = {
381 accountId: output?.accountId ?? accountId,
382 zoneId: (news.zoneId as string | undefined) ?? undefined,
383 };
384 const name = yield* createJobName(id, news.name);
385 const destinationConf = news.destinationConf as string;
386 const body = buildMutableBody(news, name, destinationConf);
387
388 // 1. Observe — by cached id first, falling back to a list scan by
389 // name so we recover from lost state or out-of-band deletes.
390 let observed: logpush.GetJobResponse | undefined;
391 if (output?.jobId !== undefined) {
392 observed = yield* observeById(scope, output.jobId);
393 }
394 let justCreated = false;
395 if (!observed) {
396 observed = yield* findByName(scope, name, news.dataset);
397 }
398
399 // 2. Ensure — create when missing. Logpush job names are not
400 // unique, so there is no AlreadyExists race to tolerate; create
401 // failures are real (e.g. invalid destination).
402 if (!observed) {
403 observed = yield* createJob(scope, news.dataset, body);
404 justCreated = true;
405 }
406
407 const jobId = observed.id;
408 if (jobId === undefined || jobId === null) {
409 return yield* new JobUnexpectedResponse({
410 message: "Cloudflare did not return an id for the Logpush job",
411 });
412 }
413
414 // 3. Sync — Cloudflare's update endpoint is PUT-style; resend the
415 // full desired body when anything differs. `destination_conf`
416 // and `filter` are compared against `olds` because Cloudflare
417 // redacts embedded secrets (destination) or omits the field
418 // entirely (filter) in get/list responses.
419 if (!justCreated && needsUpdate(body, news, olds, observed)) {
420 observed = yield* updateJob(scope, jobId, body);
421 }
422
423 const attrs = toAttributes({ ...observed, id: jobId }, scope);
424 if (!attrs) {
425 return yield* new JobUnexpectedResponse({
426 message: "Cloudflare returned a Logpush job without id/dataset/destination",
427 });
428 }
429 // Prefer the desired destination — the observed echo redacts secrets.
430 return { ...attrs, destinationConf, name: attrs.name || name };
431 }),
432
433 delete: Effect.fn(function* ({ output }) {
434 const scope: Scope = {
435 accountId: output.accountId,
436 zoneId: output.zoneId,
437 };
438 yield* (
439 scope.zoneId !== undefined
440 ? logpush.deleteJobForZone({
441 zoneId: scope.zoneId,
442 jobId: output.jobId,
443 })
444 : logpush.deleteJobForAccount({
445 accountId: scope.accountId,
446 jobId: output.jobId,
447 })
448 ).pipe(Effect.catchTag("JobNotFound", () => Effect.void));
449 }),
450 });
451
452 interface Scope {
453 accountId: string;
454 zoneId: string | undefined;
455 }
456
457 const createJobName = (id: string, name: string | undefined) =>
458 Effect.gen(function* () {
459 if (name) return name;
460 return yield* createPhysicalName({ id, lowercase: true });
461 });
462
463 const observeById = (scope: Scope, jobId: number) =>
464 (scope.zoneId !== undefined
465 ? logpush.getJobForZone({ zoneId: scope.zoneId, jobId })
466 : logpush.getJobForAccount({ accountId: scope.accountId, jobId })
467 ).pipe(
468 Effect.map((job): logpush.GetJobResponse | undefined => job),
469 Effect.catchTag("JobNotFound", () => Effect.undefined),
470 );
471
472 const findByName = (scope: Scope, name: string, dataset: Dataset | undefined) =>
473 (scope.zoneId !== undefined
474 ? logpush.listJobsForZone.items({ zoneId: scope.zoneId })
475 : logpush.listJobsForAccount.items({ accountId: scope.accountId })
476 ).pipe(
477 Stream.runCollect,
478 Effect.map((chunk): logpush.GetJobResponse | undefined => {
479 const match = Array.from(chunk)
480 .filter((job): job is NonNullable<typeof job> => job != null)
481 .find((job) => job.name === name && (dataset === undefined || job.dataset === dataset));
482 return match ?? undefined;
483 }),
484 );
485
486 /**
487 * The mutable body shared by create and update (PUT) requests.
488 */
489 interface JobMutableBody {
490 destinationConf: string;
491 enabled: boolean;
492 filter: string | undefined;
493 kind: "" | "edge" | undefined;
494 maxUploadBytes: number | undefined;
495 maxUploadIntervalSeconds: number | undefined;
496 maxUploadRecords: number | undefined;
497 name: string;
498 outputOptions: OutputOptions | undefined;
499 ownershipChallenge: string | undefined;
500 }
501
502 const buildMutableBody = (
503 news: JobProps,
504 name: string,
505 destinationConf: string,
506 ): JobMutableBody => ({
507 destinationConf,
508 enabled: news.enabled ?? false,
509 filter: news.filter,
510 kind: news.kind,
511 maxUploadBytes: news.maxUploadBytes,
512 maxUploadIntervalSeconds: news.maxUploadIntervalSeconds,
513 maxUploadRecords: news.maxUploadRecords,
514 name,
515 outputOptions: news.outputOptions,
516 ownershipChallenge: news.ownershipChallenge,
517 });
518
519 const createJob = (scope: Scope, dataset: Dataset, body: JobMutableBody) =>
520 (scope.zoneId !== undefined
521 ? logpush.createJobForZone({ zoneId: scope.zoneId, dataset, ...body })
522 : logpush.createJobForAccount({
523 accountId: scope.accountId,
524 dataset,
525 ...body,
526 })
527 ).pipe(Effect.map((job): logpush.GetJobResponse => job));
528
529 const updateJob = (scope: Scope, jobId: number, body: JobMutableBody) =>
530 (scope.zoneId !== undefined
531 ? logpush.updateJobForZone({ zoneId: scope.zoneId, jobId, ...body })
532 : logpush.updateJobForAccount({
533 accountId: scope.accountId,
534 jobId,
535 ...body,
536 })
537 ).pipe(Effect.map((job): logpush.GetJobResponse => job));
538
539 const asNumber = (v: "0" | number | null | undefined): number | undefined =>
540 v == null ? undefined : typeof v === "string" ? Number(v) : v;
541
542 /**
543 * Decide whether a PUT is needed. Observable fields are diffed against the
544 * live job; write-only/redacted fields (`destination_conf`, `filter`,
545 * `ownership_challenge`) fall back to a `news` vs `olds` comparison —
546 * `olds === undefined` (adoption) forces a PUT so the job converges to the
547 * declared destination/filter.
548 */
549 const needsUpdate = (
550 desired: JobMutableBody,
551 news: JobProps,
552 olds: JobProps | undefined,
553 observed: logpush.GetJobResponse,
554 ): boolean => {
555 if (olds === undefined) return true;
556 if ((news.destinationConf as string) !== (olds.destinationConf as string)) {
557 return true;
558 }
559 if (news.filter !== olds.filter) return true;
560 if (desired.enabled !== (observed.enabled ?? false)) return true;
561 if (desired.name !== (observed.name ?? undefined)) return true;
562 if (
563 desired.maxUploadBytes !== undefined &&
564 desired.maxUploadBytes !== asNumber(observed.maxUploadBytes)
565 ) {
566 return true;
567 }
568 if (
569 desired.maxUploadIntervalSeconds !== undefined &&
570 desired.maxUploadIntervalSeconds !== asNumber(observed.maxUploadIntervalSeconds)
571 ) {
572 return true;
573 }
574 if (
575 desired.maxUploadRecords !== undefined &&
576 desired.maxUploadRecords !== asNumber(observed.maxUploadRecords)
577 ) {
578 return true;
579 }
580 if (
581 desired.outputOptions !== undefined &&
582 !outputOptionsEqual(desired.outputOptions, observed.outputOptions)
583 ) {
584 return true;
585 }
586 return false;
587 };
588
589 /**
590 * Compare only the keys the user declared — Cloudflare fills the rest with
591 * defaults we must not fight.
592 */
593 const outputOptionsEqual = (
594 desired: OutputOptions,
595 observed: logpush.GetJobResponse["outputOptions"],
596 ): boolean => {
597 const o = observed ?? {};
598 for (const key of Object.keys(desired) as (keyof OutputOptions)[]) {
599 const want = desired[key];
600 if (want === undefined) continue;
601 const have = o[key] ?? undefined;
602 if (Array.isArray(want)) {
603 if (
604 !Array.isArray(have) ||
605 want.length !== have.length ||
606 want.some((v, i) => v !== have[i])
607 ) {
608 return false;
609 }
610 } else if (want !== have) {
611 return false;
612 }
613 }
614 return true;
615 };
616
617 const undef = <T>(v: T | null | undefined): T | undefined => (v == null ? undefined : v);
618
619 const toAttributes = (
620 observed: logpush.GetJobResponse | undefined,
621 scope: Scope,
622 ): JobAttributes | undefined => {
623 if (observed?.id == null || observed.dataset == null || observed.destinationConf == null) {
624 return undefined;
625 }
626 return {
627 jobId: observed.id,
628 accountId: scope.accountId,
629 zoneId: scope.zoneId,
630 name: observed.name ?? "",
631 dataset: observed.dataset,
632 destinationConf: observed.destinationConf,
633 enabled: observed.enabled ?? false,
634 kind: observed.kind ?? "",
635 errorMessage: undef(observed.errorMessage),
636 lastComplete: undef(observed.lastComplete),
637 lastError: undef(observed.lastError),
638 };
639 };