The best way to control Claude Code, Codex and more from any computer or phone

main
ts 745 lines 24.2 KB
Raw
1 import * as pipelines from "@distilled.cloud/cloudflare/pipelines";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import { createPhysicalName } from "alchemy/PhysicalName";
5 import * as Provider from "alchemy/Provider";
6 import { Resource } from "alchemy/Resource";
7 import * as Effect from "effect/Effect";
8 import * as Predicate from "effect/Predicate";
9 import * as Redacted from "effect/Redacted";
10 import * as Schedule from "effect/Schedule";
11 import * as Stream from "effect/Stream";
12 import { CloudflareEnvironment } from "../CloudflareEnvironment.ts";
13 import type { Providers } from "../Providers.ts";
14 import type { Bucket } from "../R2/Bucket.ts";
15 import type { DataCatalog } from "../R2/DataCatalog.ts";
16
17 const TypeId = "Cloudflare.Pipelines.Sink" as const;
18 type TypeId = typeof TypeId;
19
20 /**
21 * Batching policy controlling when the sink rolls (closes and uploads)
22 * the current output file.
23 */
24 export interface SinkRollingPolicy {
25 /**
26 * Roll the file once it reaches this size in bytes.
27 */
28 fileSizeBytes?: number;
29 /**
30 * Roll the file after this many seconds without new events.
31 */
32 inactivitySeconds?: number;
33 /**
34 * Roll the file at most this many seconds after it was opened.
35 * @default 300
36 */
37 intervalSeconds?: number;
38 }
39
40 /**
41 * Configuration of an `r2` sink writing raw files to an R2 bucket.
42 */
43 export interface SinkR2Config {
44 /**
45 * The destination R2 bucket: a `Cloudflare.R2.Bucket` resource (orders the
46 * sink after the bucket) or a bucket name. The bucket must already exist.
47 */
48 bucket: string | Bucket;
49 /**
50 * R2 S3-compatible credentials the sink uses to write objects.
51 * Write-only — Cloudflare never echoes them back, and sinks cannot be
52 * updated, so changing them does NOT replace the sink (see the
53 * resource docs on rotating credentials).
54 */
55 credentials: {
56 /** R2 access key id (the API token id). */
57 accessKeyId: Redacted.Redacted<string>;
58 /** R2 secret access key (SHA-256 hex of the API token value). */
59 secretAccessKey: Redacted.Redacted<string>;
60 };
61 /**
62 * Key prefix under which output objects are written.
63 */
64 path?: string;
65 /**
66 * Time-based partitioning of output object keys.
67 */
68 partitioning?: {
69 /**
70 * strftime-style pattern, e.g. `year=%Y/month=%m/day=%d`.
71 */
72 timePattern?: string;
73 };
74 /**
75 * Naming of output files within a partition.
76 */
77 fileNaming?: {
78 /** Prefix prepended to each file name. */
79 prefix?: string;
80 /** Suffix appended to each file name. */
81 suffix?: string;
82 /**
83 * Strategy generating the unique part of each file name.
84 * @default "uuid_v7"
85 */
86 strategy?: "serial" | "uuid" | "uuid_v7" | "ulid";
87 };
88 /**
89 * When the sink rolls output files.
90 */
91 rollingPolicy?: SinkRollingPolicy;
92 /**
93 * Jurisdiction the bucket was created in (`eu`, `fedramp`), when not
94 * the default.
95 */
96 jurisdiction?: string;
97 }
98
99 /**
100 * Configuration of a catalog sink writing an Iceberg table via the Basin
101 * (R2 Data) Catalog, spelled with the bucket name.
102 */
103 export interface SinkR2DataCatalogConfig {
104 /**
105 * The R2 bucket backing the catalog: a `Cloudflare.R2.Bucket` resource or a
106 * bucket name. The bucket must already exist and have the catalog enabled.
107 */
108 bucket: string | Bucket;
109 /**
110 * Name of the Iceberg table to write.
111 */
112 tableName: string;
113 /**
114 * Catalog namespace the table lives in.
115 * @default "default"
116 */
117 namespace?: string;
118 /**
119 * Cloudflare API token with R2 Data Catalog permissions. Write-only —
120 * Cloudflare never echoes it back, and changing it does NOT replace
121 * the sink.
122 */
123 token: Redacted.Redacted<string>;
124 /**
125 * When the sink rolls output files.
126 */
127 rollingPolicy?: SinkRollingPolicy;
128 }
129
130 /**
131 * The catalog a catalog sink writes to: a `Cloudflare.Basin.Catalog`
132 * (`Cloudflare.R2.DataCatalog`) resource, which orders the sink after the
133 * catalog, or the name of the bucket the catalog is enabled on.
134 */
135 export type SinkCatalogReference = string | DataCatalog;
136
137 /**
138 * The bucket name behind a bucket or catalog reference: the string itself,
139 * or the `bucketName` attribute of a resolved `Cloudflare.R2.Bucket` or
140 * `Cloudflare.Basin.Catalog`.
141 */
142 const bucketNameOf = (ref: unknown): string =>
143 typeof ref === "string"
144 ? ref
145 : ref !== null &&
146 typeof ref === "object" &&
147 typeof (ref as { bucketName?: unknown }).bucketName === "string"
148 ? (ref as { bucketName: string }).bucketName
149 : "";
150
151 /**
152 * The Iceberg table a catalog sink writes to.
153 */
154 export interface SinkCatalogTable {
155 /**
156 * Catalog namespace the table lives in.
157 * @default "default"
158 */
159 namespace?: string;
160 /** Table name. */
161 name: string;
162 }
163
164 /**
165 * Sink type of a catalog sink. `basin_catalog` and `r2_data_catalog` are
166 * the same destination; switching between them never replaces the sink.
167 */
168 export type SinkCatalogType = "basin_catalog" | "r2_data_catalog";
169
170 /**
171 * Output file format written by the sink.
172 */
173 export type SinkFormat =
174 | {
175 /** Newline-delimited JSON output. */
176 type: "json";
177 }
178 | {
179 /** Parquet output. */
180 type: "parquet";
181 /**
182 * Compression codec.
183 * @default "zstd"
184 */
185 compression?: "uncompressed" | "snappy" | "gzip" | "zstd" | "lz4";
186 /** Target row-group size in bytes. */
187 rowGroupBytes?: number;
188 };
189
190 interface SinkBaseProps {
191 /**
192 * Name of the sink. Unique per account; must be alphanumeric and
193 * underscores only (it is referenced as a SQL table name). If omitted,
194 * a unique name is generated from the app, stage, and logical ID.
195 *
196 * Sinks have no update API, so changing this (or any destination
197 * property) triggers a replacement.
198 * @default ${app}_${id}_${stage}_${suffix}
199 */
200 name?: string;
201 /**
202 * Output file format. Catalog sinks require Parquet.
203 * @default { type: "json" } for `r2` sinks, { type: "parquet" } for catalog sinks
204 */
205 format?: SinkFormat;
206 }
207
208 export type SinkProps =
209 | (SinkBaseProps & {
210 /**
211 * Sink type — `r2` writes raw files to an R2 bucket.
212 */
213 type: "r2";
214 /**
215 * R2 destination configuration.
216 */
217 config: SinkR2Config;
218 })
219 | (SinkBaseProps & {
220 /**
221 * Sink type — writes an Iceberg table via the Basin (R2 Data)
222 * Catalog.
223 */
224 type: SinkCatalogType;
225 /**
226 * Catalog destination configuration, spelled with the bucket name.
227 * Equivalent to the `catalog` / `table` / `token` form.
228 */
229 config: SinkR2DataCatalogConfig;
230 })
231 | (SinkBaseProps & {
232 /**
233 * Sink type — writes an Iceberg table via the Basin (R2 Data)
234 * Catalog.
235 */
236 type: SinkCatalogType;
237 /**
238 * The catalog to write to: a `Cloudflare.Basin.Catalog` resource or
239 * the name of the bucket the catalog is enabled on.
240 */
241 catalog: SinkCatalogReference;
242 /**
243 * The Iceberg table to write. Cloudflare creates it with the
244 * sink — sinks cannot be created for existing Iceberg tables.
245 */
246 table: SinkCatalogTable;
247 /**
248 * Cloudflare API token with R2 Data Catalog permissions. Write-only;
249 * changing it does NOT replace the sink.
250 */
251 token: Redacted.Redacted<string>;
252 /**
253 * When the sink rolls output files.
254 */
255 rollingPolicy?: SinkRollingPolicy;
256 });
257
258 export interface SinkAttributes {
259 /** Cloudflare-assigned sink identifier. */
260 sinkId: string;
261 /** Account that owns the sink. */
262 accountId: string;
263 /** Sink name (unique per account). */
264 name: string;
265 /** Sink type. */
266 type: "r2" | SinkCatalogType;
267 /** Destination R2 bucket name. */
268 bucket: string;
269 /** Key prefix output objects are written under (r2 sinks). */
270 path: string | undefined;
271 /** Catalog namespace of the destination table (catalog sinks). */
272 namespace: string | undefined;
273 /** Destination Iceberg table name (catalog sinks). */
274 tableName: string | undefined;
275 /** When the sink was created. */
276 createdAt: string;
277 /** When the sink was last modified. */
278 modifiedAt: string;
279 }
280
281 export type Sink = Resource<TypeId, SinkProps, SinkAttributes, never, Providers>;
282
283 /**
284 * A Cloudflare Pipelines sink — the destination of the Pipelines product
285 * (also exported as `Cloudflare.Basin.Sink`). A SQL {@link Pipeline}
286 * reads events from a {@link Stream} and writes them to a sink, which
287 * stores them in R2 either as raw files (`r2`) or as an Iceberg table via
288 * the Basin Catalog (`basin_catalog`, a.k.a. `r2_data_catalog`).
289 *
290 * Sinks have no update API: changing the destination (bucket, path,
291 * table, format, partitioning, rolling policy) triggers a replacement.
292 * Equivalent spellings never replace — `basin_catalog` ⇄
293 * `r2_data_catalog`, the `config` form ⇄ the `catalog` / `table` form,
294 * omitted ⇄ explicit defaults. With engine-generated names a replacement
295 * is seamless (the new sink gets a fresh name before the old one is
296 * deleted); with an explicit `name` it collides, so prefer generated
297 * names.
298 *
299 * Credentials (`credentials`, `token`) are write-only and are not part of
300 * the replacement diff, so rotating them never recreates the sink — a
301 * catalog sink could not be recreated anyway, because Cloudflare refuses
302 * to create sinks for existing Iceberg tables. The sink keeps the
303 * credentials it was created with; to move it onto new ones, rename it
304 * (which replaces it).
305 *
306 * Pipelines is ingest-only — to react to files the sink writes, subscribe
307 * to its bucket with `Cloudflare.R2.BucketEventNotification`.
308 *
309 * ### Creating a Sink
310 * **Example:** R2 sink with JSON output
311 * The S3-compatible credentials are derived from a Cloudflare API token:
312 * the access key id is the token id and the secret is the SHA-256 hex
313 * digest of the token value.
314 * ```typescript
315 * const bucket = yield* Cloudflare.R2.Bucket("events", {});
316 *
317 * const sink = yield* Cloudflare.Basin.Sink("events-sink", {
318 * type: "r2",
319 * config: {
320 * bucket,
321 * credentials: {
322 * accessKeyId: yield* Config.Redacted("R2_ACCESS_KEY_ID"),
323 * secretAccessKey: yield* Config.Redacted("R2_SECRET_ACCESS_KEY"),
324 * },
325 * path: "ingest",
326 * rollingPolicy: { intervalSeconds: 30 },
327 * },
328 * });
329 * ```
330 *
331 * **Example:** Parquet output
332 * ```typescript
333 * const sink = yield* Cloudflare.Basin.Sink("parquet-sink", {
334 * type: "r2",
335 * config: { bucket, credentials },
336 * format: { type: "parquet", compression: "zstd" },
337 * });
338 * ```
339 *
340 * ### Basin Catalog
341 * **Example:** Iceberg table sink
342 * ```typescript
343 * const bucket = yield* Cloudflare.R2.Bucket("Lakehouse", {});
344 * const catalog = yield* Cloudflare.Basin.Catalog("Catalog", {
345 * bucket,
346 * });
347 *
348 * const sink = yield* Cloudflare.Basin.Sink("PageViewTable", {
349 * type: "basin_catalog",
350 * catalog,
351 * table: { namespace: "web", name: "page_views" },
352 * token: yield* Config.Redacted("CATALOG_TOKEN"),
353 * });
354 * ```
355 *
356 * **Example:** Bucket-name form
357 * ```typescript
358 * const sink = yield* Cloudflare.Basin.Sink("iceberg-sink", {
359 * type: "r2_data_catalog",
360 * config: {
361 * bucket,
362 * tableName: "events",
363 * namespace: "default",
364 * token: yield* Config.Redacted("CATALOG_TOKEN"),
365 * },
366 * });
367 * ```
368 *
369 * @see https://developers.cloudflare.com/pipelines/
370 *
371 * @resource
372 * @product Pipelines
373 * @category Storage & Databases
374 */
375 export const Sink = Resource<Sink>(TypeId);
376
377 /**
378 * Returns true if the given value is a Sink resource.
379 */
380 export const isSink = (value: unknown): value is Sink =>
381 Predicate.hasProperty(value, "Type") && value.Type === TypeId;
382
383 export const SinkProvider = () =>
384 Provider.succeed(Sink, {
385 stables: ["sinkId", "accountId", "name", "type", "createdAt"],
386
387 diff: Effect.fn(function* ({ id, olds, news, output }) {
388 const { accountId } = yield* yield* CloudflareEnvironment;
389 if (!isResolved(news)) return undefined;
390 if ((output?.accountId ?? accountId) !== accountId) {
391 return { action: "replace" } as const;
392 }
393 const o = olds as SinkProps | undefined;
394 if (o === undefined) return undefined;
395 // Sinks have no update API — any destination change replaces. Compare
396 // the CANONICAL destination (credentials excluded) so equivalent
397 // spellings and credential rotation never replace.
398 const newName = yield* sinkName(id, news.name);
399 const oldName = output?.name ?? (yield* sinkName(id, o.name));
400 if (newName !== oldName) {
401 return { action: "replace" } as const;
402 }
403 if (!stableEquals(canonicalSink(news), canonicalSink(o))) {
404 return { action: "replace" } as const;
405 }
406 return { action: "noop" } as const;
407 }),
408
409 read: Effect.fn(function* ({ id, output, olds }) {
410 const { accountId } = yield* yield* CloudflareEnvironment;
411 const acct = output?.accountId ?? accountId;
412 if (output?.sinkId) {
413 const observed = yield* getSink(acct, output.sinkId);
414 if (observed) return toAttributes(observed, acct);
415 }
416 // Cold read — sink names are unique per account; a generated-name
417 // match is proof of ownership, an explicit name is not.
418 const name = yield* sinkName(id, olds?.name);
419 const match = yield* findSinkByName(acct, name);
420 if (match) {
421 const attrs = toAttributes(match, acct);
422 return olds?.name !== undefined ? Unowned(attrs) : attrs;
423 }
424 return undefined;
425 }),
426
427 reconcile: Effect.fn(function* ({ id, news, output }) {
428 const { accountId } = yield* yield* CloudflareEnvironment;
429 const name = yield* sinkName(id, news.name);
430
431 // 1. Observe — by cached id first, then by (unique) name so we
432 // recover from lost state writes.
433 let observed = output?.sinkId
434 ? yield* getSink(output.accountId ?? accountId, output.sinkId)
435 : undefined;
436 if (!observed) {
437 observed = yield* findSinkByName(accountId, name);
438 }
439
440 // Converge drift — there is no update API, so when observable
441 // echoed config (bucket/path/table) differs from the desired state
442 // (e.g. the bucket reference was unresolved at diff time), delete
443 // the stale sink and fall through to recreate it under the same
444 // name. The delete rides out `SinkInUse` while a pipeline that is
445 // being repointed still references it.
446 if (observed && sinkDrifted(observed, news)) {
447 yield* deleteSink(accountId, observed.id);
448 // Wait until the delete is visible so the recreate below does not
449 // race a `SinkAlreadyExists` against the dying sink.
450 yield* getSink(accountId, observed.id).pipe(
451 Effect.repeat({
452 schedule: Schedule.max([Schedule.exponential("250 millis"), Schedule.recurs(8)]),
453 until: (s) => s === undefined,
454 }),
455 );
456 observed = undefined;
457 }
458
459 // 2. Ensure — create when missing. There is no sync step: sinks
460 // have no update API, so destination changes arrive as
461 // replacements (diff) rather than in-place updates. An
462 // AlreadyExists on create is a race or recovery — resolve it via
463 // the name lookup.
464 if (!observed) {
465 observed = yield* pipelines
466 .createSink({
467 accountId,
468 name,
469 type: news.type,
470 config: toRequestConfig(accountId, news),
471 format: news.format ?? defaultFormat(news.type),
472 })
473 .pipe(
474 Effect.catchTag("SinkAlreadyExists", (error) =>
475 findSinkByName(accountId, name).pipe(
476 Effect.flatMap((match) => (match ? Effect.succeed(match) : Effect.fail(error))),
477 ),
478 ),
479 );
480 }
481
482 return toAttributes(observed, accountId);
483 }),
484
485 delete: Effect.fn(function* ({ output }) {
486 yield* deleteSink(output.accountId, output.sinkId);
487 }),
488
489 // Account collection: sinks are account-scoped and enumerable via
490 // `listSinks` (paginated, items in `result`). Hydrate each page item
491 // into the exact `read` Attributes shape. Credentials/token are
492 // write-only and never echoed, matching `read`.
493 list: Effect.fn(function* () {
494 const { accountId } = yield* yield* CloudflareEnvironment;
495 return yield* pipelines.listSinks.pages({ accountId }).pipe(
496 Stream.runCollect,
497 Effect.map((chunk) =>
498 Array.from(chunk).flatMap((page) =>
499 (page.result ?? []).map((sink) => toAttributes(sink, accountId)),
500 ),
501 ),
502 );
503 }),
504 });
505
506 /**
507 * The subset of sink state shared by get/list/create responses.
508 */
509 interface ObservedSink {
510 id: string;
511 name: string;
512 type: string;
513 config?:
514 | {
515 bucket: string;
516 path?: string | null;
517 }
518 | { bucket: string; tableName: string; namespace?: string | null }
519 | null;
520 createdAt: string;
521 modifiedAt: string;
522 }
523
524 /** A catalog sink destination, whichever form it was declared in. */
525 interface CatalogTarget {
526 bucket: string;
527 tableName: string;
528 namespace: string | undefined;
529 token: Redacted.Redacted<string>;
530 rollingPolicy: SinkRollingPolicy | undefined;
531 }
532
533 const isCatalogType = (type: string) => type === "r2_data_catalog" || type === "basin_catalog";
534
535 const catalogTarget = (props: Exclude<SinkProps, { type: "r2" }>): CatalogTarget =>
536 "config" in props && props.config !== undefined
537 ? {
538 bucket: bucketNameOf(props.config.bucket),
539 tableName: props.config.tableName,
540 namespace: props.config.namespace,
541 token: props.config.token,
542 rollingPolicy: props.config.rollingPolicy,
543 }
544 : {
545 bucket: bucketNameOf((props as { catalog: SinkCatalogReference }).catalog),
546 tableName: (props as { table: SinkCatalogTable }).table.name,
547 namespace: (props as { table: SinkCatalogTable }).table.namespace,
548 token: (props as { token: Redacted.Redacted<string> }).token,
549 rollingPolicy: (props as { rollingPolicy?: SinkRollingPolicy }).rollingPolicy,
550 };
551
552 // Pipelines entity names must be alphanumeric/underscore only (they are
553 // referenced as SQL table names), so swap the default hyphen delimiter
554 // for underscores.
555 const sinkName = (id: string, name: string | undefined) =>
556 Effect.gen(function* () {
557 if (name) return name;
558 const generated = yield* createPhysicalName({
559 id,
560 lowercase: true,
561 delimiter: "_",
562 });
563 return generated.replaceAll(/[^a-zA-Z0-9_]/g, "_");
564 });
565
566 /**
567 * Read a sink by id, mapping "gone" (`SinkNotFound`, Cloudflare error
568 * code 1015, or `InvalidSinkId` for a malformed/foreign id) to
569 * `undefined`.
570 */
571 const getSink = (accountId: string, sinkId: string) =>
572 pipelines.getSink({ accountId, sinkId }).pipe(
573 Effect.map((s): ObservedSink | undefined => s),
574 Effect.catchTags({
575 SinkNotFound: () => Effect.undefined,
576 InvalidSinkId: () => Effect.undefined,
577 }),
578 );
579
580 /**
581 * Idempotent delete — a sink that is already gone is success. Rides out
582 * `SinkInUse` (HTTP 422 "Sink still in use") with a bounded retry while
583 * a dependent pipeline's own deletion propagates.
584 */
585 const deleteSink = (accountId: string, sinkId: string) =>
586 pipelines.deleteSink({ accountId, sinkId }).pipe(
587 Effect.retry({
588 while: (e) => e._tag === "SinkInUse",
589 schedule: Schedule.max([Schedule.exponential("500 millis"), Schedule.recurs(8)]),
590 }),
591 Effect.catchTags({
592 SinkNotFound: () => Effect.void,
593 InvalidSinkId: () => Effect.void,
594 }),
595 );
596
597 /**
598 * Detect drift between the echoed cloud config and the desired props.
599 * Only fields Cloudflare echoes back are comparable (credentials and the
600 * catalog token are write-only); only user-declared optional fields are
601 * diffed so we don't fight server-side defaults.
602 */
603 const sinkDrifted = (observed: ObservedSink, news: SinkProps): boolean => {
604 if (isCatalogType(observed.type) !== isCatalogType(news.type)) return true;
605 const cfg = observed.config;
606 if (!cfg) return false;
607 if (news.type === "r2") {
608 if (cfg.bucket !== bucketNameOf(news.config.bucket)) return true;
609 const observedPath = "path" in cfg ? (cfg.path ?? undefined) : undefined;
610 return news.config.path !== undefined && news.config.path !== observedPath;
611 }
612 const target = catalogTarget(news);
613 if (cfg.bucket !== target.bucket) return true;
614 if ("tableName" in cfg) {
615 if (cfg.tableName !== target.tableName) return true;
616 if (target.namespace !== undefined && target.namespace !== (cfg.namespace ?? undefined)) {
617 return true;
618 }
619 }
620 return false;
621 };
622
623 const findSinkByName = (accountId: string, name: string) =>
624 pipelines.listSinks.items({ accountId }).pipe(
625 Stream.runCollect,
626 Effect.map((chunk): ObservedSink | undefined => Array.from(chunk).find((s) => s.name === name)),
627 );
628
629 /**
630 * Build the distilled create request `config` body from props,
631 * unwrapping write-only secrets.
632 */
633 const toRequestConfig = (accountId: string, news: SinkProps) => {
634 if (news.type === "r2") {
635 const c = news.config;
636 return {
637 accountId,
638 bucket: bucketNameOf(c.bucket),
639 credentials: {
640 accessKeyId: Redacted.value(c.credentials.accessKeyId),
641 secretAccessKey: Redacted.value(c.credentials.secretAccessKey),
642 },
643 path: c.path,
644 partitioning: c.partitioning,
645 fileNaming: c.fileNaming,
646 rollingPolicy: c.rollingPolicy,
647 jurisdiction: c.jurisdiction,
648 };
649 }
650 const t = catalogTarget(news);
651 return {
652 accountId,
653 bucket: t.bucket,
654 tableName: t.tableName,
655 namespace: t.namespace,
656 token: Redacted.value(t.token),
657 rollingPolicy: t.rollingPolicy,
658 };
659 };
660
661 const canonicalRollingPolicy = (rp: SinkRollingPolicy | undefined) => ({
662 fileSizeBytes: rp?.fileSizeBytes,
663 inactivitySeconds: rp?.inactivitySeconds,
664 intervalSeconds: rp?.intervalSeconds ?? 300,
665 });
666
667 /** Catalog (Iceberg) sinks only accept Parquet; raw R2 sinks default to JSON. */
668 const defaultFormat = (type: SinkProps["type"]): SinkFormat =>
669 isCatalogType(type) ? { type: "parquet" } : { type: "json" };
670
671 const canonicalFormat = (type: SinkProps["type"], format: SinkFormat | undefined) => {
672 const f = format ?? defaultFormat(type);
673 return f.type === "parquet"
674 ? {
675 type: "parquet",
676 compression: f.compression ?? "zstd",
677 rowGroupBytes: f.rowGroupBytes,
678 }
679 : { type: "json" };
680 };
681
682 /**
683 * The canonical destination of a sink, for change detection: catalog
684 * type spellings and declaration forms collapse, defaults are filled in,
685 * and write-only credentials are excluded (they cannot be updated, and
686 * rotating them must never recreate the sink).
687 */
688 const canonicalSink = (props: SinkProps): unknown => {
689 const format = canonicalFormat(props.type, props.format);
690 if (props.type === "r2") {
691 const c = props.config;
692 return {
693 kind: "r2",
694 format,
695 bucket: bucketNameOf(c.bucket),
696 path: c.path,
697 partitioning: c.partitioning,
698 fileNaming: c.fileNaming
699 ? { ...c.fileNaming, strategy: c.fileNaming.strategy ?? "uuid_v7" }
700 : undefined,
701 rollingPolicy: canonicalRollingPolicy(c.rollingPolicy),
702 jurisdiction: c.jurisdiction,
703 };
704 }
705 const t = catalogTarget(props);
706 return {
707 kind: "catalog",
708 format,
709 bucket: t.bucket,
710 namespace: t.namespace ?? "default",
711 tableName: t.tableName,
712 rollingPolicy: canonicalRollingPolicy(t.rollingPolicy),
713 };
714 };
715
716 /**
717 * Key-order-insensitive structural equality for plain JSON-ish prop
718 * values.
719 */
720 const stableEquals = (a: unknown, b: unknown): boolean => stableStringify(a) === stableStringify(b);
721
722 const stableStringify = (value: unknown): string =>
723 JSON.stringify(value, (_key, v) =>
724 v !== null && typeof v === "object" && !Array.isArray(v)
725 ? Object.fromEntries(
726 Object.entries(v as Record<string, unknown>).sort(([x], [y]) => x.localeCompare(y)),
727 )
728 : v,
729 ) ?? "undefined";
730
731 const toAttributes = (observed: ObservedSink, accountId: string): SinkAttributes => {
732 const cfg = observed.config ?? undefined;
733 return {
734 sinkId: observed.id,
735 accountId,
736 name: observed.name,
737 type: observed.type as SinkAttributes["type"],
738 bucket: cfg?.bucket ?? "",
739 path: cfg && "path" in cfg ? (cfg.path ?? undefined) : undefined,
740 namespace: cfg && "namespace" in cfg ? (cfg.namespace ?? undefined) : undefined,
741 tableName: cfg && "tableName" in cfg ? cfg.tableName : undefined,
742 createdAt: observed.createdAt,
743 modifiedAt: observed.modifiedAt,
744 };
745 };