The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import * as pipelines from "@distilled.cloud/cloudflare/pipelines"; |
| 2 | import { Unowned } from "alchemy/AdoptPolicy"; |
| 3 | import { isResolved } from "alchemy/Diff"; |
| 4 | import { createPhysicalName } from "alchemy/PhysicalName"; |
| 5 | import * as Provider from "alchemy/Provider"; |
| 6 | import { Resource } from "alchemy/Resource"; |
| 7 | import * as Data from "effect/Data"; |
| 8 | import * as Effect from "effect/Effect"; |
| 9 | import * as Predicate from "effect/Predicate"; |
| 10 | import * as Schedule from "effect/Schedule"; |
| 11 | import * as Stream from "effect/Stream"; |
| 12 | import { CloudflareEnvironment } from "../CloudflareEnvironment.ts"; |
| 13 | import type { Providers } from "../Providers.ts"; |
| 14 | |
| 15 | const TypeId = "Cloudflare.Pipelines.Pipeline" as const; |
| 16 | type TypeId = typeof TypeId; |
| 17 | |
| 18 | export interface PipelineProps { |
| 19 | /** |
| 20 | * Name of the pipeline. Unique per account; must be alphanumeric and |
| 21 | * underscores only. If omitted, a unique name is generated from the |
| 22 | * app, stage, and logical ID. |
| 23 | * |
| 24 | * Pipelines have no update API, so changing this (or the `sql`) |
| 25 | * triggers a replacement. |
| 26 | * @default ${app}_${id}_${stage}_${suffix} |
| 27 | */ |
| 28 | name?: string; |
| 29 | /** |
| 30 | * SQL statement describing the processing flow, e.g. |
| 31 | * `INSERT INTO my_sink SELECT * FROM my_stream`. Streams and sinks are |
| 32 | * referenced by name — interpolate `stream.name` / `sink.name` outputs |
| 33 | * so the engine orders the pipeline after them on deploy (and before |
| 34 | * them on destroy). |
| 35 | * |
| 36 | * Immutable — changing the SQL triggers a replacement. Whitespace-only |
| 37 | * changes are a no-op. A changed, fully-resolved statement is validated |
| 38 | * during plan, so invalid SQL fails before anything is replaced. |
| 39 | */ |
| 40 | sql: string; |
| 41 | } |
| 42 | |
| 43 | /** |
| 44 | * Raised during plan when a pipeline's SQL is rejected by Cloudflare's |
| 45 | * SQL validator (`InvalidSql`, code 1014). Nothing has been replaced or |
| 46 | * deleted when this fails. |
| 47 | */ |
| 48 | export class PipelineSqlInvalid extends Data.TaggedError( |
| 49 | "Cloudflare.Pipelines.PipelineSqlInvalid", |
| 50 | )<{ |
| 51 | message: string; |
| 52 | sql: string; |
| 53 | }> {} |
| 54 | |
| 55 | export interface PipelineAttributes { |
| 56 | /** Cloudflare-assigned pipeline identifier. */ |
| 57 | pipelineId: string; |
| 58 | /** Account that owns the pipeline. */ |
| 59 | accountId: string; |
| 60 | /** Pipeline name (unique per account). */ |
| 61 | name: string; |
| 62 | /** SQL statement of the processing flow. */ |
| 63 | sql: string; |
| 64 | /** Current status of the pipeline. */ |
| 65 | status: string; |
| 66 | /** When the pipeline was created. */ |
| 67 | createdAt: string; |
| 68 | /** When the pipeline was last modified. */ |
| 69 | modifiedAt: string; |
| 70 | } |
| 71 | |
| 72 | export type Pipeline = Resource<TypeId, PipelineProps, PipelineAttributes, never, Providers>; |
| 73 | |
| 74 | /** |
| 75 | * A Cloudflare SQL Pipeline — the transform of the Pipelines product. A |
| 76 | * pipeline is a single SQL statement that reads events from a |
| 77 | * {@link Stream} and writes them to a {@link Sink}, both |
| 78 | * referenced by name. |
| 79 | * |
| 80 | * Also exported as `Cloudflare.Basin.Pipeline`. |
| 81 | * |
| 82 | * The SQL is fixed at creation: changing it (or the name) triggers a |
| 83 | * replacement. Nothing references a pipeline downstream, so replacements |
| 84 | * are cheap. A changed statement is checked with Cloudflare's SQL |
| 85 | * validator during plan, so a syntax error fails the plan (with |
| 86 | * {@link PipelineSqlInvalid}) instead of deleting the running pipeline. |
| 87 | * References to streams/sinks that do not exist yet are not an error at |
| 88 | * plan time — they are created in the same deploy. |
| 89 | * |
| 90 | * Pipelines is ingest-only: to process data after the sink writes it, |
| 91 | * subscribe to the sink's bucket with `Cloudflare.R2.BucketEventNotification`. |
| 92 | * ### Creating a Pipeline |
| 93 | * **Example:** Stream → Sink passthrough |
| 94 | * ```typescript |
| 95 | * const stream = yield* Cloudflare.Basin.Stream("events", {}); |
| 96 | * const sink = yield* Cloudflare.Basin.Sink("events-sink", { |
| 97 | * type: "r2", |
| 98 | * config: { bucket: bucket.bucketName, credentials }, |
| 99 | * }); |
| 100 | * |
| 101 | * const pipeline = yield* Cloudflare.Basin.Pipeline("etl", { |
| 102 | * sql: Output.interpolate`INSERT INTO ${sink.name} SELECT * FROM ${stream.name}`, |
| 103 | * }); |
| 104 | * ``` |
| 105 | * |
| 106 | * **Example:** Filtering transform |
| 107 | * ```typescript |
| 108 | * const pipeline = yield* Cloudflare.Basin.Pipeline("errors-only", { |
| 109 | * sql: Output.interpolate`INSERT INTO ${sink.name} SELECT * FROM ${stream.name} WHERE level = 'error'`, |
| 110 | * }); |
| 111 | * ``` |
| 112 | * |
| 113 | * @see https://developers.cloudflare.com/pipelines/ |
| 114 | * |
| 115 | * @resource |
| 116 | * @product Pipelines |
| 117 | * @category Storage & Databases |
| 118 | */ |
| 119 | export const Pipeline = Resource<Pipeline>(TypeId); |
| 120 | |
| 121 | /** |
| 122 | * Returns true if the given value is a Pipeline resource. |
| 123 | */ |
| 124 | export const isPipeline = (value: unknown): value is Pipeline => |
| 125 | Predicate.hasProperty(value, "Type") && value.Type === TypeId; |
| 126 | |
| 127 | export const PipelineProvider = () => |
| 128 | Provider.succeed(Pipeline, { |
| 129 | stables: ["pipelineId", "accountId", "name", "createdAt"], |
| 130 | |
| 131 | diff: Effect.fn(function* ({ id, olds, news, output }) { |
| 132 | const { accountId } = yield* yield* CloudflareEnvironment; |
| 133 | if (!isResolved(news)) return undefined; |
| 134 | if ((output?.accountId ?? accountId) !== accountId) { |
| 135 | return { action: "replace" } as const; |
| 136 | } |
| 137 | const o = olds as PipelineProps | undefined; |
| 138 | if (o === undefined) return undefined; |
| 139 | // Pipelines have no update API — name and sql changes replace. |
| 140 | const newName = yield* pipelineName(id, news.name); |
| 141 | const oldName = output?.name ?? (yield* pipelineName(id, o.name)); |
| 142 | if (newName !== oldName) { |
| 143 | return { action: "replace" } as const; |
| 144 | } |
| 145 | const oldSql = typeof o.sql === "string" ? o.sql : output?.sql; |
| 146 | if (oldSql === undefined || canonicalSql(news.sql) === canonicalSql(oldSql)) { |
| 147 | return oldSql === undefined ? undefined : ({ action: "noop" } as const); |
| 148 | } |
| 149 | // The replacement deletes the running pipeline — validate the new |
| 150 | // statement first so bad SQL fails the plan, not the deploy. Tables |
| 151 | // created in the same deploy are not visible yet, so a missing table |
| 152 | // is not a validation failure. |
| 153 | yield* pipelines.validateSqlPipeline({ accountId, sql: news.sql }).pipe( |
| 154 | Effect.catchTags({ |
| 155 | TableNotFound: () => Effect.void, |
| 156 | InvalidSql: (error) => |
| 157 | Effect.fail( |
| 158 | new PipelineSqlInvalid({ |
| 159 | message: `Cloudflare.Pipelines.Pipeline '${id}': invalid SQL — ${error.message}`, |
| 160 | sql: news.sql, |
| 161 | }), |
| 162 | ), |
| 163 | }), |
| 164 | ); |
| 165 | return { action: "replace" } as const; |
| 166 | }), |
| 167 | |
| 168 | read: Effect.fn(function* ({ id, output, olds }) { |
| 169 | const { accountId } = yield* yield* CloudflareEnvironment; |
| 170 | const acct = output?.accountId ?? accountId; |
| 171 | if (output?.pipelineId) { |
| 172 | const observed = yield* getPipeline(acct, output.pipelineId); |
| 173 | if (observed) return toAttributes(observed, acct); |
| 174 | } |
| 175 | // Cold read — pipeline names are unique per account; a |
| 176 | // generated-name match is proof of ownership, an explicit name is |
| 177 | // not. |
| 178 | const name = yield* pipelineName(id, olds?.name); |
| 179 | const match = yield* findPipelineByName(acct, name); |
| 180 | if (match) { |
| 181 | const attrs = toAttributes(match, acct); |
| 182 | return olds?.name !== undefined ? Unowned(attrs) : attrs; |
| 183 | } |
| 184 | return undefined; |
| 185 | }), |
| 186 | |
| 187 | reconcile: Effect.fn(function* ({ id, news, output }) { |
| 188 | const { accountId } = yield* yield* CloudflareEnvironment; |
| 189 | const name = yield* pipelineName(id, news.name); |
| 190 | const sql = news.sql as string; |
| 191 | |
| 192 | // 1. Observe — by cached id first, then by (unique) name so we |
| 193 | // recover from lost state writes. |
| 194 | let observed = output?.pipelineId |
| 195 | ? yield* getPipeline(output.accountId ?? accountId, output.pipelineId) |
| 196 | : undefined; |
| 197 | if (!observed) { |
| 198 | observed = yield* findPipelineByName(accountId, name); |
| 199 | } |
| 200 | |
| 201 | // Converge drift — there is no update API, so when the observed SQL |
| 202 | // differs from the desired SQL (e.g. a referenced stream/sink was |
| 203 | // replaced and `sql` was unresolved at diff time), delete the stale |
| 204 | // pipeline and fall through to recreate it under the same name. |
| 205 | // Nothing references a pipeline downstream, so this is safe. |
| 206 | if ( |
| 207 | observed && |
| 208 | (canonicalSql(observed.sql) !== canonicalSql(sql) || observed.name !== name) |
| 209 | ) { |
| 210 | yield* deletePipeline(accountId, observed.id); |
| 211 | // Wait until the delete is visible so the recreate below does not |
| 212 | // race an `PipelineAlreadyExists` against the dying pipeline. |
| 213 | yield* getPipeline(accountId, observed.id).pipe( |
| 214 | Effect.repeat({ |
| 215 | schedule: Schedule.max([Schedule.exponential("250 millis"), Schedule.recurs(8)]), |
| 216 | until: (p) => p === undefined, |
| 217 | }), |
| 218 | ); |
| 219 | observed = undefined; |
| 220 | } |
| 221 | |
| 222 | // 2. Ensure — create when missing. The referenced stream/sink may |
| 223 | // have been created moments ago, so ride out eventual-consistency |
| 224 | // `TableNotFound` blips with a bounded retry. An AlreadyExists is |
| 225 | // a race or recovery — resolve it via the name lookup. There is |
| 226 | // no sync step: the SQL is immutable, so changes arrive as |
| 227 | // replacements (diff). |
| 228 | if (!observed) { |
| 229 | observed = yield* pipelines.createV1Pipeline({ accountId, name, sql }).pipe( |
| 230 | Effect.retry({ |
| 231 | while: (e) => e._tag === "TableNotFound", |
| 232 | schedule: Schedule.max([Schedule.exponential("500 millis"), Schedule.recurs(6)]), |
| 233 | }), |
| 234 | Effect.catchTag("PipelineAlreadyExists", (error) => |
| 235 | findPipelineByName(accountId, name).pipe( |
| 236 | Effect.flatMap((match) => (match ? Effect.succeed(match) : Effect.fail(error))), |
| 237 | ), |
| 238 | ), |
| 239 | ); |
| 240 | } |
| 241 | |
| 242 | return toAttributes(observed, accountId); |
| 243 | }), |
| 244 | |
| 245 | delete: Effect.fn(function* ({ output }) { |
| 246 | yield* deletePipeline(output.accountId, output.pipelineId); |
| 247 | }), |
| 248 | |
| 249 | // Account collection — pipelines are account-scoped. Exhaustively |
| 250 | // paginate the account-wide list and hydrate each row into the same |
| 251 | // Attributes shape `read` returns. |
| 252 | list: Effect.fn(function* () { |
| 253 | const { accountId } = yield* yield* CloudflareEnvironment; |
| 254 | return yield* pipelines.listV1Pipeline.pages({ accountId }).pipe( |
| 255 | Stream.runCollect, |
| 256 | Effect.map((chunk) => |
| 257 | Array.from(chunk).flatMap((page) => |
| 258 | (page.result ?? []).map((p) => toAttributes(p, accountId)), |
| 259 | ), |
| 260 | ), |
| 261 | ); |
| 262 | }), |
| 263 | }); |
| 264 | |
| 265 | /** |
| 266 | * Canonical SQL for change detection: whitespace runs outside quoted |
| 267 | * literals/identifiers collapse to one space, surrounding whitespace and |
| 268 | * a trailing `;` are dropped. |
| 269 | */ |
| 270 | const canonicalSql = (sql: string): string => { |
| 271 | let out = ""; |
| 272 | let quote: string | undefined; |
| 273 | let pendingSpace = false; |
| 274 | for (const ch of sql.trim().replace(/;\s*$/, "")) { |
| 275 | if (quote) { |
| 276 | out += ch; |
| 277 | if (ch === quote) quote = undefined; |
| 278 | } else if (/\s/.test(ch)) { |
| 279 | pendingSpace = true; |
| 280 | } else { |
| 281 | if (pendingSpace && out.length > 0) out += " "; |
| 282 | pendingSpace = false; |
| 283 | if (ch === "'" || ch === '"' || ch === "`") quote = ch; |
| 284 | out += ch; |
| 285 | } |
| 286 | } |
| 287 | return out; |
| 288 | }; |
| 289 | |
| 290 | /** |
| 291 | * The subset of pipeline state shared by get/list/create responses. |
| 292 | */ |
| 293 | interface ObservedPipeline { |
| 294 | id: string; |
| 295 | name: string; |
| 296 | sql: string; |
| 297 | status: string; |
| 298 | createdAt: string; |
| 299 | modifiedAt: string; |
| 300 | } |
| 301 | |
| 302 | // Pipelines entity names must be alphanumeric/underscore only, so swap |
| 303 | // the default hyphen delimiter for underscores. |
| 304 | const pipelineName = (id: string, name: string | undefined) => |
| 305 | Effect.gen(function* () { |
| 306 | if (name) return name; |
| 307 | const generated = yield* createPhysicalName({ |
| 308 | id, |
| 309 | lowercase: true, |
| 310 | delimiter: "_", |
| 311 | }); |
| 312 | return generated.replaceAll(/[^a-zA-Z0-9_]/g, "_"); |
| 313 | }); |
| 314 | |
| 315 | /** |
| 316 | * Read a pipeline by id, mapping "gone" (`PipelineNotExists`, Cloudflare |
| 317 | * error code 1000) to `undefined`. |
| 318 | */ |
| 319 | const getPipeline = (accountId: string, pipelineId: string) => |
| 320 | pipelines.getV1Pipeline({ accountId, pipelineId }).pipe( |
| 321 | Effect.map((p): ObservedPipeline | undefined => p), |
| 322 | Effect.catchTag("PipelineNotExists", () => Effect.undefined), |
| 323 | ); |
| 324 | |
| 325 | /** |
| 326 | * Idempotent delete — a pipeline that is already gone is success. |
| 327 | */ |
| 328 | const deletePipeline = (accountId: string, pipelineId: string) => |
| 329 | pipelines |
| 330 | .deleteV1Pipeline({ accountId, pipelineId }) |
| 331 | .pipe(Effect.catchTag("PipelineNotExists", () => Effect.void)); |
| 332 | |
| 333 | const findPipelineByName = (accountId: string, name: string) => |
| 334 | pipelines.listV1Pipeline.items({ accountId }).pipe( |
| 335 | Stream.runCollect, |
| 336 | Effect.map((chunk): ObservedPipeline | undefined => |
| 337 | Array.from(chunk).find((p) => p.name === name), |
| 338 | ), |
| 339 | ); |
| 340 | |
| 341 | const toAttributes = (observed: ObservedPipeline, accountId: string): PipelineAttributes => ({ |
| 342 | pipelineId: observed.id, |
| 343 | accountId, |
| 344 | name: observed.name, |
| 345 | sql: observed.sql, |
| 346 | status: observed.status, |
| 347 | createdAt: observed.createdAt, |
| 348 | modifiedAt: observed.modifiedAt, |
| 349 | }); |