The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 06:00 UTC
| 1 | import * as pipelines from "@distilled.cloud/cloudflare/pipelines"; |
| 2 | import { Unowned } from "alchemy/AdoptPolicy"; |
| 3 | import { isResolved } from "alchemy/Diff"; |
| 4 | import { createPhysicalName } from "alchemy/PhysicalName"; |
| 5 | import * as Provider from "alchemy/Provider"; |
| 6 | import { Resource } from "alchemy/Resource"; |
| 7 | import * as Effect from "effect/Effect"; |
| 8 | import * as Predicate from "effect/Predicate"; |
| 9 | import * as Redacted from "effect/Redacted"; |
| 10 | import * as Schedule from "effect/Schedule"; |
| 11 | import * as Stream from "effect/Stream"; |
| 12 | import { CloudflareEnvironment } from "../CloudflareEnvironment.ts"; |
| 13 | import type { Providers } from "../Providers.ts"; |
| 14 | import type { Bucket } from "../R2/Bucket.ts"; |
| 15 | import type { DataCatalog } from "../R2/DataCatalog.ts"; |
| 16 | |
| 17 | const TypeId = "Cloudflare.Pipelines.Sink" as const; |
| 18 | type TypeId = typeof TypeId; |
| 19 | |
| 20 | /** |
| 21 | * Batching policy controlling when the sink rolls (closes and uploads) |
| 22 | * the current output file. |
| 23 | */ |
| 24 | export interface SinkRollingPolicy { |
| 25 | /** |
| 26 | * Roll the file once it reaches this size in bytes. |
| 27 | */ |
| 28 | fileSizeBytes?: number; |
| 29 | /** |
| 30 | * Roll the file after this many seconds without new events. |
| 31 | */ |
| 32 | inactivitySeconds?: number; |
| 33 | /** |
| 34 | * Roll the file at most this many seconds after it was opened. |
| 35 | * @default 300 |
| 36 | */ |
| 37 | intervalSeconds?: number; |
| 38 | } |
| 39 | |
| 40 | /** |
| 41 | * Configuration of an `r2` sink writing raw files to an R2 bucket. |
| 42 | */ |
| 43 | export interface SinkR2Config { |
| 44 | /** |
| 45 | * The destination R2 bucket: a `Cloudflare.R2.Bucket` resource (orders the |
| 46 | * sink after the bucket) or a bucket name. The bucket must already exist. |
| 47 | */ |
| 48 | bucket: string | Bucket; |
| 49 | /** |
| 50 | * R2 S3-compatible credentials the sink uses to write objects. |
| 51 | * Write-only — Cloudflare never echoes them back, and sinks cannot be |
| 52 | * updated, so changing them does NOT replace the sink (see the |
| 53 | * resource docs on rotating credentials). |
| 54 | */ |
| 55 | credentials: { |
| 56 | /** R2 access key id (the API token id). */ |
| 57 | accessKeyId: Redacted.Redacted<string>; |
| 58 | /** R2 secret access key (SHA-256 hex of the API token value). */ |
| 59 | secretAccessKey: Redacted.Redacted<string>; |
| 60 | }; |
| 61 | /** |
| 62 | * Key prefix under which output objects are written. |
| 63 | */ |
| 64 | path?: string; |
| 65 | /** |
| 66 | * Time-based partitioning of output object keys. |
| 67 | */ |
| 68 | partitioning?: { |
| 69 | /** |
| 70 | * strftime-style pattern, e.g. `year=%Y/month=%m/day=%d`. |
| 71 | */ |
| 72 | timePattern?: string; |
| 73 | }; |
| 74 | /** |
| 75 | * Naming of output files within a partition. |
| 76 | */ |
| 77 | fileNaming?: { |
| 78 | /** Prefix prepended to each file name. */ |
| 79 | prefix?: string; |
| 80 | /** Suffix appended to each file name. */ |
| 81 | suffix?: string; |
| 82 | /** |
| 83 | * Strategy generating the unique part of each file name. |
| 84 | * @default "uuid_v7" |
| 85 | */ |
| 86 | strategy?: "serial" | "uuid" | "uuid_v7" | "ulid"; |
| 87 | }; |
| 88 | /** |
| 89 | * When the sink rolls output files. |
| 90 | */ |
| 91 | rollingPolicy?: SinkRollingPolicy; |
| 92 | /** |
| 93 | * Jurisdiction the bucket was created in (`eu`, `fedramp`), when not |
| 94 | * the default. |
| 95 | */ |
| 96 | jurisdiction?: string; |
| 97 | } |
| 98 | |
| 99 | /** |
| 100 | * Configuration of a catalog sink writing an Iceberg table via the Basin |
| 101 | * (R2 Data) Catalog, spelled with the bucket name. |
| 102 | */ |
| 103 | export interface SinkR2DataCatalogConfig { |
| 104 | /** |
| 105 | * The R2 bucket backing the catalog: a `Cloudflare.R2.Bucket` resource or a |
| 106 | * bucket name. The bucket must already exist and have the catalog enabled. |
| 107 | */ |
| 108 | bucket: string | Bucket; |
| 109 | /** |
| 110 | * Name of the Iceberg table to write. |
| 111 | */ |
| 112 | tableName: string; |
| 113 | /** |
| 114 | * Catalog namespace the table lives in. |
| 115 | * @default "default" |
| 116 | */ |
| 117 | namespace?: string; |
| 118 | /** |
| 119 | * Cloudflare API token with R2 Data Catalog permissions. Write-only — |
| 120 | * Cloudflare never echoes it back, and changing it does NOT replace |
| 121 | * the sink. |
| 122 | */ |
| 123 | token: Redacted.Redacted<string>; |
| 124 | /** |
| 125 | * When the sink rolls output files. |
| 126 | */ |
| 127 | rollingPolicy?: SinkRollingPolicy; |
| 128 | } |
| 129 | |
| 130 | /** |
| 131 | * The catalog a catalog sink writes to: a `Cloudflare.Basin.Catalog` |
| 132 | * (`Cloudflare.R2.DataCatalog`) resource, which orders the sink after the |
| 133 | * catalog, or the name of the bucket the catalog is enabled on. |
| 134 | */ |
| 135 | export type SinkCatalogReference = string | DataCatalog; |
| 136 | |
| 137 | /** |
| 138 | * The bucket name behind a bucket or catalog reference: the string itself, |
| 139 | * or the `bucketName` attribute of a resolved `Cloudflare.R2.Bucket` or |
| 140 | * `Cloudflare.Basin.Catalog`. |
| 141 | */ |
| 142 | const bucketNameOf = (ref: unknown): string => |
| 143 | typeof ref === "string" |
| 144 | ? ref |
| 145 | : ref !== null && |
| 146 | typeof ref === "object" && |
| 147 | typeof (ref as { bucketName?: unknown }).bucketName === "string" |
| 148 | ? (ref as { bucketName: string }).bucketName |
| 149 | : ""; |
| 150 | |
| 151 | /** |
| 152 | * The Iceberg table a catalog sink writes to. |
| 153 | */ |
| 154 | export interface SinkCatalogTable { |
| 155 | /** |
| 156 | * Catalog namespace the table lives in. |
| 157 | * @default "default" |
| 158 | */ |
| 159 | namespace?: string; |
| 160 | /** Table name. */ |
| 161 | name: string; |
| 162 | } |
| 163 | |
| 164 | /** |
| 165 | * Sink type of a catalog sink. `basin_catalog` and `r2_data_catalog` are |
| 166 | * the same destination; switching between them never replaces the sink. |
| 167 | */ |
| 168 | export type SinkCatalogType = "basin_catalog" | "r2_data_catalog"; |
| 169 | |
| 170 | /** |
| 171 | * Output file format written by the sink. |
| 172 | */ |
| 173 | export type SinkFormat = |
| 174 | | { |
| 175 | /** Newline-delimited JSON output. */ |
| 176 | type: "json"; |
| 177 | } |
| 178 | | { |
| 179 | /** Parquet output. */ |
| 180 | type: "parquet"; |
| 181 | /** |
| 182 | * Compression codec. |
| 183 | * @default "zstd" |
| 184 | */ |
| 185 | compression?: "uncompressed" | "snappy" | "gzip" | "zstd" | "lz4"; |
| 186 | /** Target row-group size in bytes. */ |
| 187 | rowGroupBytes?: number; |
| 188 | }; |
| 189 | |
| 190 | interface SinkBaseProps { |
| 191 | /** |
| 192 | * Name of the sink. Unique per account; must be alphanumeric and |
| 193 | * underscores only (it is referenced as a SQL table name). If omitted, |
| 194 | * a unique name is generated from the app, stage, and logical ID. |
| 195 | * |
| 196 | * Sinks have no update API, so changing this (or any destination |
| 197 | * property) triggers a replacement. |
| 198 | * @default ${app}_${id}_${stage}_${suffix} |
| 199 | */ |
| 200 | name?: string; |
| 201 | /** |
| 202 | * Output file format. Catalog sinks require Parquet. |
| 203 | * @default { type: "json" } for `r2` sinks, { type: "parquet" } for catalog sinks |
| 204 | */ |
| 205 | format?: SinkFormat; |
| 206 | } |
| 207 | |
| 208 | export type SinkProps = |
| 209 | | (SinkBaseProps & { |
| 210 | /** |
| 211 | * Sink type — `r2` writes raw files to an R2 bucket. |
| 212 | */ |
| 213 | type: "r2"; |
| 214 | /** |
| 215 | * R2 destination configuration. |
| 216 | */ |
| 217 | config: SinkR2Config; |
| 218 | }) |
| 219 | | (SinkBaseProps & { |
| 220 | /** |
| 221 | * Sink type — writes an Iceberg table via the Basin (R2 Data) |
| 222 | * Catalog. |
| 223 | */ |
| 224 | type: SinkCatalogType; |
| 225 | /** |
| 226 | * Catalog destination configuration, spelled with the bucket name. |
| 227 | * Equivalent to the `catalog` / `table` / `token` form. |
| 228 | */ |
| 229 | config: SinkR2DataCatalogConfig; |
| 230 | }) |
| 231 | | (SinkBaseProps & { |
| 232 | /** |
| 233 | * Sink type — writes an Iceberg table via the Basin (R2 Data) |
| 234 | * Catalog. |
| 235 | */ |
| 236 | type: SinkCatalogType; |
| 237 | /** |
| 238 | * The catalog to write to: a `Cloudflare.Basin.Catalog` resource or |
| 239 | * the name of the bucket the catalog is enabled on. |
| 240 | */ |
| 241 | catalog: SinkCatalogReference; |
| 242 | /** |
| 243 | * The Iceberg table to write. Cloudflare creates it with the |
| 244 | * sink — sinks cannot be created for existing Iceberg tables. |
| 245 | */ |
| 246 | table: SinkCatalogTable; |
| 247 | /** |
| 248 | * Cloudflare API token with R2 Data Catalog permissions. Write-only; |
| 249 | * changing it does NOT replace the sink. |
| 250 | */ |
| 251 | token: Redacted.Redacted<string>; |
| 252 | /** |
| 253 | * When the sink rolls output files. |
| 254 | */ |
| 255 | rollingPolicy?: SinkRollingPolicy; |
| 256 | }); |
| 257 | |
| 258 | export interface SinkAttributes { |
| 259 | /** Cloudflare-assigned sink identifier. */ |
| 260 | sinkId: string; |
| 261 | /** Account that owns the sink. */ |
| 262 | accountId: string; |
| 263 | /** Sink name (unique per account). */ |
| 264 | name: string; |
| 265 | /** Sink type. */ |
| 266 | type: "r2" | SinkCatalogType; |
| 267 | /** Destination R2 bucket name. */ |
| 268 | bucket: string; |
| 269 | /** Key prefix output objects are written under (r2 sinks). */ |
| 270 | path: string | undefined; |
| 271 | /** Catalog namespace of the destination table (catalog sinks). */ |
| 272 | namespace: string | undefined; |
| 273 | /** Destination Iceberg table name (catalog sinks). */ |
| 274 | tableName: string | undefined; |
| 275 | /** When the sink was created. */ |
| 276 | createdAt: string; |
| 277 | /** When the sink was last modified. */ |
| 278 | modifiedAt: string; |
| 279 | } |
| 280 | |
| 281 | export type Sink = Resource<TypeId, SinkProps, SinkAttributes, never, Providers>; |
| 282 | |
| 283 | /** |
| 284 | * A Cloudflare Pipelines sink — the destination of the Pipelines product |
| 285 | * (also exported as `Cloudflare.Basin.Sink`). A SQL {@link Pipeline} |
| 286 | * reads events from a {@link Stream} and writes them to a sink, which |
| 287 | * stores them in R2 either as raw files (`r2`) or as an Iceberg table via |
| 288 | * the Basin Catalog (`basin_catalog`, a.k.a. `r2_data_catalog`). |
| 289 | * |
| 290 | * Sinks have no update API: changing the destination (bucket, path, |
| 291 | * table, format, partitioning, rolling policy) triggers a replacement. |
| 292 | * Equivalent spellings never replace — `basin_catalog` ⇄ |
| 293 | * `r2_data_catalog`, the `config` form ⇄ the `catalog` / `table` form, |
| 294 | * omitted ⇄ explicit defaults. With engine-generated names a replacement |
| 295 | * is seamless (the new sink gets a fresh name before the old one is |
| 296 | * deleted); with an explicit `name` it collides, so prefer generated |
| 297 | * names. |
| 298 | * |
| 299 | * Credentials (`credentials`, `token`) are write-only and are not part of |
| 300 | * the replacement diff, so rotating them never recreates the sink — a |
| 301 | * catalog sink could not be recreated anyway, because Cloudflare refuses |
| 302 | * to create sinks for existing Iceberg tables. The sink keeps the |
| 303 | * credentials it was created with; to move it onto new ones, rename it |
| 304 | * (which replaces it). |
| 305 | * |
| 306 | * Pipelines is ingest-only — to react to files the sink writes, subscribe |
| 307 | * to its bucket with `Cloudflare.R2.BucketEventNotification`. |
| 308 | * |
| 309 | * ### Creating a Sink |
| 310 | * **Example:** R2 sink with JSON output |
| 311 | * The S3-compatible credentials are derived from a Cloudflare API token: |
| 312 | * the access key id is the token id and the secret is the SHA-256 hex |
| 313 | * digest of the token value. |
| 314 | * ```typescript |
| 315 | * const bucket = yield* Cloudflare.R2.Bucket("events", {}); |
| 316 | * |
| 317 | * const sink = yield* Cloudflare.Basin.Sink("events-sink", { |
| 318 | * type: "r2", |
| 319 | * config: { |
| 320 | * bucket, |
| 321 | * credentials: { |
| 322 | * accessKeyId: yield* Config.Redacted("R2_ACCESS_KEY_ID"), |
| 323 | * secretAccessKey: yield* Config.Redacted("R2_SECRET_ACCESS_KEY"), |
| 324 | * }, |
| 325 | * path: "ingest", |
| 326 | * rollingPolicy: { intervalSeconds: 30 }, |
| 327 | * }, |
| 328 | * }); |
| 329 | * ``` |
| 330 | * |
| 331 | * **Example:** Parquet output |
| 332 | * ```typescript |
| 333 | * const sink = yield* Cloudflare.Basin.Sink("parquet-sink", { |
| 334 | * type: "r2", |
| 335 | * config: { bucket, credentials }, |
| 336 | * format: { type: "parquet", compression: "zstd" }, |
| 337 | * }); |
| 338 | * ``` |
| 339 | * |
| 340 | * ### Basin Catalog |
| 341 | * **Example:** Iceberg table sink |
| 342 | * ```typescript |
| 343 | * const bucket = yield* Cloudflare.R2.Bucket("Lakehouse", {}); |
| 344 | * const catalog = yield* Cloudflare.Basin.Catalog("Catalog", { |
| 345 | * bucket, |
| 346 | * }); |
| 347 | * |
| 348 | * const sink = yield* Cloudflare.Basin.Sink("PageViewTable", { |
| 349 | * type: "basin_catalog", |
| 350 | * catalog, |
| 351 | * table: { namespace: "web", name: "page_views" }, |
| 352 | * token: yield* Config.Redacted("CATALOG_TOKEN"), |
| 353 | * }); |
| 354 | * ``` |
| 355 | * |
| 356 | * **Example:** Bucket-name form |
| 357 | * ```typescript |
| 358 | * const sink = yield* Cloudflare.Basin.Sink("iceberg-sink", { |
| 359 | * type: "r2_data_catalog", |
| 360 | * config: { |
| 361 | * bucket, |
| 362 | * tableName: "events", |
| 363 | * namespace: "default", |
| 364 | * token: yield* Config.Redacted("CATALOG_TOKEN"), |
| 365 | * }, |
| 366 | * }); |
| 367 | * ``` |
| 368 | * |
| 369 | * @see https://developers.cloudflare.com/pipelines/ |
| 370 | * |
| 371 | * @resource |
| 372 | * @product Pipelines |
| 373 | * @category Storage & Databases |
| 374 | */ |
| 375 | export const Sink = Resource<Sink>(TypeId); |
| 376 | |
| 377 | /** |
| 378 | * Returns true if the given value is a Sink resource. |
| 379 | */ |
| 380 | export const isSink = (value: unknown): value is Sink => |
| 381 | Predicate.hasProperty(value, "Type") && value.Type === TypeId; |
| 382 | |
| 383 | export const SinkProvider = () => |
| 384 | Provider.succeed(Sink, { |
| 385 | stables: ["sinkId", "accountId", "name", "type", "createdAt"], |
| 386 | |
| 387 | diff: Effect.fn(function* ({ id, olds, news, output }) { |
| 388 | const { accountId } = yield* yield* CloudflareEnvironment; |
| 389 | if (!isResolved(news)) return undefined; |
| 390 | if ((output?.accountId ?? accountId) !== accountId) { |
| 391 | return { action: "replace" } as const; |
| 392 | } |
| 393 | const o = olds as SinkProps | undefined; |
| 394 | if (o === undefined) return undefined; |
| 395 | // Sinks have no update API — any destination change replaces. Compare |
| 396 | // the CANONICAL destination (credentials excluded) so equivalent |
| 397 | // spellings and credential rotation never replace. |
| 398 | const newName = yield* sinkName(id, news.name); |
| 399 | const oldName = output?.name ?? (yield* sinkName(id, o.name)); |
| 400 | if (newName !== oldName) { |
| 401 | return { action: "replace" } as const; |
| 402 | } |
| 403 | if (!stableEquals(canonicalSink(news), canonicalSink(o))) { |
| 404 | return { action: "replace" } as const; |
| 405 | } |
| 406 | return { action: "noop" } as const; |
| 407 | }), |
| 408 | |
| 409 | read: Effect.fn(function* ({ id, output, olds }) { |
| 410 | const { accountId } = yield* yield* CloudflareEnvironment; |
| 411 | const acct = output?.accountId ?? accountId; |
| 412 | if (output?.sinkId) { |
| 413 | const observed = yield* getSink(acct, output.sinkId); |
| 414 | if (observed) return toAttributes(observed, acct); |
| 415 | } |
| 416 | // Cold read — sink names are unique per account; a generated-name |
| 417 | // match is proof of ownership, an explicit name is not. |
| 418 | const name = yield* sinkName(id, olds?.name); |
| 419 | const match = yield* findSinkByName(acct, name); |
| 420 | if (match) { |
| 421 | const attrs = toAttributes(match, acct); |
| 422 | return olds?.name !== undefined ? Unowned(attrs) : attrs; |
| 423 | } |
| 424 | return undefined; |
| 425 | }), |
| 426 | |
| 427 | reconcile: Effect.fn(function* ({ id, news, output }) { |
| 428 | const { accountId } = yield* yield* CloudflareEnvironment; |
| 429 | const name = yield* sinkName(id, news.name); |
| 430 | |
| 431 | // 1. Observe — by cached id first, then by (unique) name so we |
| 432 | // recover from lost state writes. |
| 433 | let observed = output?.sinkId |
| 434 | ? yield* getSink(output.accountId ?? accountId, output.sinkId) |
| 435 | : undefined; |
| 436 | if (!observed) { |
| 437 | observed = yield* findSinkByName(accountId, name); |
| 438 | } |
| 439 | |
| 440 | // Converge drift — there is no update API, so when observable |
| 441 | // echoed config (bucket/path/table) differs from the desired state |
| 442 | // (e.g. the bucket reference was unresolved at diff time), delete |
| 443 | // the stale sink and fall through to recreate it under the same |
| 444 | // name. The delete rides out `SinkInUse` while a pipeline that is |
| 445 | // being repointed still references it. |
| 446 | if (observed && sinkDrifted(observed, news)) { |
| 447 | yield* deleteSink(accountId, observed.id); |
| 448 | // Wait until the delete is visible so the recreate below does not |
| 449 | // race a `SinkAlreadyExists` against the dying sink. |
| 450 | yield* getSink(accountId, observed.id).pipe( |
| 451 | Effect.repeat({ |
| 452 | schedule: Schedule.max([Schedule.exponential("250 millis"), Schedule.recurs(8)]), |
| 453 | until: (s) => s === undefined, |
| 454 | }), |
| 455 | ); |
| 456 | observed = undefined; |
| 457 | } |
| 458 | |
| 459 | // 2. Ensure — create when missing. There is no sync step: sinks |
| 460 | // have no update API, so destination changes arrive as |
| 461 | // replacements (diff) rather than in-place updates. An |
| 462 | // AlreadyExists on create is a race or recovery — resolve it via |
| 463 | // the name lookup. |
| 464 | if (!observed) { |
| 465 | observed = yield* pipelines |
| 466 | .createSink({ |
| 467 | accountId, |
| 468 | name, |
| 469 | type: news.type, |
| 470 | config: toRequestConfig(accountId, news), |
| 471 | format: news.format ?? defaultFormat(news.type), |
| 472 | }) |
| 473 | .pipe( |
| 474 | Effect.catchTag("SinkAlreadyExists", (error) => |
| 475 | findSinkByName(accountId, name).pipe( |
| 476 | Effect.flatMap((match) => (match ? Effect.succeed(match) : Effect.fail(error))), |
| 477 | ), |
| 478 | ), |
| 479 | ); |
| 480 | } |
| 481 | |
| 482 | return toAttributes(observed, accountId); |
| 483 | }), |
| 484 | |
| 485 | delete: Effect.fn(function* ({ output }) { |
| 486 | yield* deleteSink(output.accountId, output.sinkId); |
| 487 | }), |
| 488 | |
| 489 | // Account collection: sinks are account-scoped and enumerable via |
| 490 | // `listSinks` (paginated, items in `result`). Hydrate each page item |
| 491 | // into the exact `read` Attributes shape. Credentials/token are |
| 492 | // write-only and never echoed, matching `read`. |
| 493 | list: Effect.fn(function* () { |
| 494 | const { accountId } = yield* yield* CloudflareEnvironment; |
| 495 | return yield* pipelines.listSinks.pages({ accountId }).pipe( |
| 496 | Stream.runCollect, |
| 497 | Effect.map((chunk) => |
| 498 | Array.from(chunk).flatMap((page) => |
| 499 | (page.result ?? []).map((sink) => toAttributes(sink, accountId)), |
| 500 | ), |
| 501 | ), |
| 502 | ); |
| 503 | }), |
| 504 | }); |
| 505 | |
| 506 | /** |
| 507 | * The subset of sink state shared by get/list/create responses. |
| 508 | */ |
| 509 | interface ObservedSink { |
| 510 | id: string; |
| 511 | name: string; |
| 512 | type: string; |
| 513 | config?: |
| 514 | | { |
| 515 | bucket: string; |
| 516 | path?: string | null; |
| 517 | } |
| 518 | | { bucket: string; tableName: string; namespace?: string | null } |
| 519 | | null; |
| 520 | createdAt: string; |
| 521 | modifiedAt: string; |
| 522 | } |
| 523 | |
| 524 | /** A catalog sink destination, whichever form it was declared in. */ |
| 525 | interface CatalogTarget { |
| 526 | bucket: string; |
| 527 | tableName: string; |
| 528 | namespace: string | undefined; |
| 529 | token: Redacted.Redacted<string>; |
| 530 | rollingPolicy: SinkRollingPolicy | undefined; |
| 531 | } |
| 532 | |
| 533 | const isCatalogType = (type: string) => type === "r2_data_catalog" || type === "basin_catalog"; |
| 534 | |
| 535 | const catalogTarget = (props: Exclude<SinkProps, { type: "r2" }>): CatalogTarget => |
| 536 | "config" in props && props.config !== undefined |
| 537 | ? { |
| 538 | bucket: bucketNameOf(props.config.bucket), |
| 539 | tableName: props.config.tableName, |
| 540 | namespace: props.config.namespace, |
| 541 | token: props.config.token, |
| 542 | rollingPolicy: props.config.rollingPolicy, |
| 543 | } |
| 544 | : { |
| 545 | bucket: bucketNameOf((props as { catalog: SinkCatalogReference }).catalog), |
| 546 | tableName: (props as { table: SinkCatalogTable }).table.name, |
| 547 | namespace: (props as { table: SinkCatalogTable }).table.namespace, |
| 548 | token: (props as { token: Redacted.Redacted<string> }).token, |
| 549 | rollingPolicy: (props as { rollingPolicy?: SinkRollingPolicy }).rollingPolicy, |
| 550 | }; |
| 551 | |
| 552 | // Pipelines entity names must be alphanumeric/underscore only (they are |
| 553 | // referenced as SQL table names), so swap the default hyphen delimiter |
| 554 | // for underscores. |
| 555 | const sinkName = (id: string, name: string | undefined) => |
| 556 | Effect.gen(function* () { |
| 557 | if (name) return name; |
| 558 | const generated = yield* createPhysicalName({ |
| 559 | id, |
| 560 | lowercase: true, |
| 561 | delimiter: "_", |
| 562 | }); |
| 563 | return generated.replaceAll(/[^a-zA-Z0-9_]/g, "_"); |
| 564 | }); |
| 565 | |
| 566 | /** |
| 567 | * Read a sink by id, mapping "gone" (`SinkNotFound`, Cloudflare error |
| 568 | * code 1015, or `InvalidSinkId` for a malformed/foreign id) to |
| 569 | * `undefined`. |
| 570 | */ |
| 571 | const getSink = (accountId: string, sinkId: string) => |
| 572 | pipelines.getSink({ accountId, sinkId }).pipe( |
| 573 | Effect.map((s): ObservedSink | undefined => s), |
| 574 | Effect.catchTags({ |
| 575 | SinkNotFound: () => Effect.undefined, |
| 576 | InvalidSinkId: () => Effect.undefined, |
| 577 | }), |
| 578 | ); |
| 579 | |
| 580 | /** |
| 581 | * Idempotent delete — a sink that is already gone is success. Rides out |
| 582 | * `SinkInUse` (HTTP 422 "Sink still in use") with a bounded retry while |
| 583 | * a dependent pipeline's own deletion propagates. |
| 584 | */ |
| 585 | const deleteSink = (accountId: string, sinkId: string) => |
| 586 | pipelines.deleteSink({ accountId, sinkId }).pipe( |
| 587 | Effect.retry({ |
| 588 | while: (e) => e._tag === "SinkInUse", |
| 589 | schedule: Schedule.max([Schedule.exponential("500 millis"), Schedule.recurs(8)]), |
| 590 | }), |
| 591 | Effect.catchTags({ |
| 592 | SinkNotFound: () => Effect.void, |
| 593 | InvalidSinkId: () => Effect.void, |
| 594 | }), |
| 595 | ); |
| 596 | |
| 597 | /** |
| 598 | * Detect drift between the echoed cloud config and the desired props. |
| 599 | * Only fields Cloudflare echoes back are comparable (credentials and the |
| 600 | * catalog token are write-only); only user-declared optional fields are |
| 601 | * diffed so we don't fight server-side defaults. |
| 602 | */ |
| 603 | const sinkDrifted = (observed: ObservedSink, news: SinkProps): boolean => { |
| 604 | if (isCatalogType(observed.type) !== isCatalogType(news.type)) return true; |
| 605 | const cfg = observed.config; |
| 606 | if (!cfg) return false; |
| 607 | if (news.type === "r2") { |
| 608 | if (cfg.bucket !== bucketNameOf(news.config.bucket)) return true; |
| 609 | const observedPath = "path" in cfg ? (cfg.path ?? undefined) : undefined; |
| 610 | return news.config.path !== undefined && news.config.path !== observedPath; |
| 611 | } |
| 612 | const target = catalogTarget(news); |
| 613 | if (cfg.bucket !== target.bucket) return true; |
| 614 | if ("tableName" in cfg) { |
| 615 | if (cfg.tableName !== target.tableName) return true; |
| 616 | if (target.namespace !== undefined && target.namespace !== (cfg.namespace ?? undefined)) { |
| 617 | return true; |
| 618 | } |
| 619 | } |
| 620 | return false; |
| 621 | }; |
| 622 | |
| 623 | const findSinkByName = (accountId: string, name: string) => |
| 624 | pipelines.listSinks.items({ accountId }).pipe( |
| 625 | Stream.runCollect, |
| 626 | Effect.map((chunk): ObservedSink | undefined => Array.from(chunk).find((s) => s.name === name)), |
| 627 | ); |
| 628 | |
| 629 | /** |
| 630 | * Build the distilled create request `config` body from props, |
| 631 | * unwrapping write-only secrets. |
| 632 | */ |
| 633 | const toRequestConfig = (accountId: string, news: SinkProps) => { |
| 634 | if (news.type === "r2") { |
| 635 | const c = news.config; |
| 636 | return { |
| 637 | accountId, |
| 638 | bucket: bucketNameOf(c.bucket), |
| 639 | credentials: { |
| 640 | accessKeyId: Redacted.value(c.credentials.accessKeyId), |
| 641 | secretAccessKey: Redacted.value(c.credentials.secretAccessKey), |
| 642 | }, |
| 643 | path: c.path, |
| 644 | partitioning: c.partitioning, |
| 645 | fileNaming: c.fileNaming, |
| 646 | rollingPolicy: c.rollingPolicy, |
| 647 | jurisdiction: c.jurisdiction, |
| 648 | }; |
| 649 | } |
| 650 | const t = catalogTarget(news); |
| 651 | return { |
| 652 | accountId, |
| 653 | bucket: t.bucket, |
| 654 | tableName: t.tableName, |
| 655 | namespace: t.namespace, |
| 656 | token: Redacted.value(t.token), |
| 657 | rollingPolicy: t.rollingPolicy, |
| 658 | }; |
| 659 | }; |
| 660 | |
| 661 | const canonicalRollingPolicy = (rp: SinkRollingPolicy | undefined) => ({ |
| 662 | fileSizeBytes: rp?.fileSizeBytes, |
| 663 | inactivitySeconds: rp?.inactivitySeconds, |
| 664 | intervalSeconds: rp?.intervalSeconds ?? 300, |
| 665 | }); |
| 666 | |
| 667 | /** Catalog (Iceberg) sinks only accept Parquet; raw R2 sinks default to JSON. */ |
| 668 | const defaultFormat = (type: SinkProps["type"]): SinkFormat => |
| 669 | isCatalogType(type) ? { type: "parquet" } : { type: "json" }; |
| 670 | |
| 671 | const canonicalFormat = (type: SinkProps["type"], format: SinkFormat | undefined) => { |
| 672 | const f = format ?? defaultFormat(type); |
| 673 | return f.type === "parquet" |
| 674 | ? { |
| 675 | type: "parquet", |
| 676 | compression: f.compression ?? "zstd", |
| 677 | rowGroupBytes: f.rowGroupBytes, |
| 678 | } |
| 679 | : { type: "json" }; |
| 680 | }; |
| 681 | |
| 682 | /** |
| 683 | * The canonical destination of a sink, for change detection: catalog |
| 684 | * type spellings and declaration forms collapse, defaults are filled in, |
| 685 | * and write-only credentials are excluded (they cannot be updated, and |
| 686 | * rotating them must never recreate the sink). |
| 687 | */ |
| 688 | const canonicalSink = (props: SinkProps): unknown => { |
| 689 | const format = canonicalFormat(props.type, props.format); |
| 690 | if (props.type === "r2") { |
| 691 | const c = props.config; |
| 692 | return { |
| 693 | kind: "r2", |
| 694 | format, |
| 695 | bucket: bucketNameOf(c.bucket), |
| 696 | path: c.path, |
| 697 | partitioning: c.partitioning, |
| 698 | fileNaming: c.fileNaming |
| 699 | ? { ...c.fileNaming, strategy: c.fileNaming.strategy ?? "uuid_v7" } |
| 700 | : undefined, |
| 701 | rollingPolicy: canonicalRollingPolicy(c.rollingPolicy), |
| 702 | jurisdiction: c.jurisdiction, |
| 703 | }; |
| 704 | } |
| 705 | const t = catalogTarget(props); |
| 706 | return { |
| 707 | kind: "catalog", |
| 708 | format, |
| 709 | bucket: t.bucket, |
| 710 | namespace: t.namespace ?? "default", |
| 711 | tableName: t.tableName, |
| 712 | rollingPolicy: canonicalRollingPolicy(t.rollingPolicy), |
| 713 | }; |
| 714 | }; |
| 715 | |
| 716 | /** |
| 717 | * Key-order-insensitive structural equality for plain JSON-ish prop |
| 718 | * values. |
| 719 | */ |
| 720 | const stableEquals = (a: unknown, b: unknown): boolean => stableStringify(a) === stableStringify(b); |
| 721 | |
| 722 | const stableStringify = (value: unknown): string => |
| 723 | JSON.stringify(value, (_key, v) => |
| 724 | v !== null && typeof v === "object" && !Array.isArray(v) |
| 725 | ? Object.fromEntries( |
| 726 | Object.entries(v as Record<string, unknown>).sort(([x], [y]) => x.localeCompare(y)), |
| 727 | ) |
| 728 | : v, |
| 729 | ) ?? "undefined"; |
| 730 | |
| 731 | const toAttributes = (observed: ObservedSink, accountId: string): SinkAttributes => { |
| 732 | const cfg = observed.config ?? undefined; |
| 733 | return { |
| 734 | sinkId: observed.id, |
| 735 | accountId, |
| 736 | name: observed.name, |
| 737 | type: observed.type as SinkAttributes["type"], |
| 738 | bucket: cfg?.bucket ?? "", |
| 739 | path: cfg && "path" in cfg ? (cfg.path ?? undefined) : undefined, |
| 740 | namespace: cfg && "namespace" in cfg ? (cfg.namespace ?? undefined) : undefined, |
| 741 | tableName: cfg && "tableName" in cfg ? cfg.tableName : undefined, |
| 742 | createdAt: observed.createdAt, |
| 743 | modifiedAt: observed.modifiedAt, |
| 744 | }; |
| 745 | }; |