@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 349 lines 12.6 KB
Raw
1 import * as pipelines from "@distilled.cloud/cloudflare/pipelines";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import { createPhysicalName } from "alchemy/PhysicalName";
5 import * as Provider from "alchemy/Provider";
6 import { Resource } from "alchemy/Resource";
7 import * as Data from "effect/Data";
8 import * as Effect from "effect/Effect";
9 import * as Predicate from "effect/Predicate";
10 import * as Schedule from "effect/Schedule";
11 import * as Stream from "effect/Stream";
12 import { CloudflareEnvironment } from "../CloudflareEnvironment.ts";
13 import type { Providers } from "../Providers.ts";
14
15 const TypeId = "Cloudflare.Pipelines.Pipeline" as const;
16 type TypeId = typeof TypeId;
17
18 export interface PipelineProps {
19 /**
20 * Name of the pipeline. Unique per account; must be alphanumeric and
21 * underscores only. If omitted, a unique name is generated from the
22 * app, stage, and logical ID.
23 *
24 * Pipelines have no update API, so changing this (or the `sql`)
25 * triggers a replacement.
26 * @default ${app}_${id}_${stage}_${suffix}
27 */
28 name?: string;
29 /**
30 * SQL statement describing the processing flow, e.g.
31 * `INSERT INTO my_sink SELECT * FROM my_stream`. Streams and sinks are
32 * referenced by name — interpolate `stream.name` / `sink.name` outputs
33 * so the engine orders the pipeline after them on deploy (and before
34 * them on destroy).
35 *
36 * Immutable — changing the SQL triggers a replacement. Whitespace-only
37 * changes are a no-op. A changed, fully-resolved statement is validated
38 * during plan, so invalid SQL fails before anything is replaced.
39 */
40 sql: string;
41 }
42
43 /**
44 * Raised during plan when a pipeline's SQL is rejected by Cloudflare's
45 * SQL validator (`InvalidSql`, code 1014). Nothing has been replaced or
46 * deleted when this fails.
47 */
48 export class PipelineSqlInvalid extends Data.TaggedError(
49 "Cloudflare.Pipelines.PipelineSqlInvalid",
50 )<{
51 message: string;
52 sql: string;
53 }> {}
54
55 export interface PipelineAttributes {
56 /** Cloudflare-assigned pipeline identifier. */
57 pipelineId: string;
58 /** Account that owns the pipeline. */
59 accountId: string;
60 /** Pipeline name (unique per account). */
61 name: string;
62 /** SQL statement of the processing flow. */
63 sql: string;
64 /** Current status of the pipeline. */
65 status: string;
66 /** When the pipeline was created. */
67 createdAt: string;
68 /** When the pipeline was last modified. */
69 modifiedAt: string;
70 }
71
72 export type Pipeline = Resource<TypeId, PipelineProps, PipelineAttributes, never, Providers>;
73
74 /**
75 * A Cloudflare SQL Pipeline — the transform of the Pipelines product. A
76 * pipeline is a single SQL statement that reads events from a
77 * {@link Stream} and writes them to a {@link Sink}, both
78 * referenced by name.
79 *
80 * Also exported as `Cloudflare.Basin.Pipeline`.
81 *
82 * The SQL is fixed at creation: changing it (or the name) triggers a
83 * replacement. Nothing references a pipeline downstream, so replacements
84 * are cheap. A changed statement is checked with Cloudflare's SQL
85 * validator during plan, so a syntax error fails the plan (with
86 * {@link PipelineSqlInvalid}) instead of deleting the running pipeline.
87 * References to streams/sinks that do not exist yet are not an error at
88 * plan time — they are created in the same deploy.
89 *
90 * Pipelines is ingest-only: to process data after the sink writes it,
91 * subscribe to the sink's bucket with `Cloudflare.R2.BucketEventNotification`.
92 * ### Creating a Pipeline
93 * **Example:** Stream → Sink passthrough
94 * ```typescript
95 * const stream = yield* Cloudflare.Basin.Stream("events", {});
96 * const sink = yield* Cloudflare.Basin.Sink("events-sink", {
97 * type: "r2",
98 * config: { bucket: bucket.bucketName, credentials },
99 * });
100 *
101 * const pipeline = yield* Cloudflare.Basin.Pipeline("etl", {
102 * sql: Output.interpolate`INSERT INTO ${sink.name} SELECT * FROM ${stream.name}`,
103 * });
104 * ```
105 *
106 * **Example:** Filtering transform
107 * ```typescript
108 * const pipeline = yield* Cloudflare.Basin.Pipeline("errors-only", {
109 * sql: Output.interpolate`INSERT INTO ${sink.name} SELECT * FROM ${stream.name} WHERE level = 'error'`,
110 * });
111 * ```
112 *
113 * @see https://developers.cloudflare.com/pipelines/
114 *
115 * @resource
116 * @product Pipelines
117 * @category Storage & Databases
118 */
119 export const Pipeline = Resource<Pipeline>(TypeId);
120
121 /**
122 * Returns true if the given value is a Pipeline resource.
123 */
124 export const isPipeline = (value: unknown): value is Pipeline =>
125 Predicate.hasProperty(value, "Type") && value.Type === TypeId;
126
127 export const PipelineProvider = () =>
128 Provider.succeed(Pipeline, {
129 stables: ["pipelineId", "accountId", "name", "createdAt"],
130
131 diff: Effect.fn(function* ({ id, olds, news, output }) {
132 const { accountId } = yield* yield* CloudflareEnvironment;
133 if (!isResolved(news)) return undefined;
134 if ((output?.accountId ?? accountId) !== accountId) {
135 return { action: "replace" } as const;
136 }
137 const o = olds as PipelineProps | undefined;
138 if (o === undefined) return undefined;
139 // Pipelines have no update API — name and sql changes replace.
140 const newName = yield* pipelineName(id, news.name);
141 const oldName = output?.name ?? (yield* pipelineName(id, o.name));
142 if (newName !== oldName) {
143 return { action: "replace" } as const;
144 }
145 const oldSql = typeof o.sql === "string" ? o.sql : output?.sql;
146 if (oldSql === undefined || canonicalSql(news.sql) === canonicalSql(oldSql)) {
147 return oldSql === undefined ? undefined : ({ action: "noop" } as const);
148 }
149 // The replacement deletes the running pipeline — validate the new
150 // statement first so bad SQL fails the plan, not the deploy. Tables
151 // created in the same deploy are not visible yet, so a missing table
152 // is not a validation failure.
153 yield* pipelines.validateSqlPipeline({ accountId, sql: news.sql }).pipe(
154 Effect.catchTags({
155 TableNotFound: () => Effect.void,
156 InvalidSql: (error) =>
157 Effect.fail(
158 new PipelineSqlInvalid({
159 message: `Cloudflare.Pipelines.Pipeline '${id}': invalid SQL — ${error.message}`,
160 sql: news.sql,
161 }),
162 ),
163 }),
164 );
165 return { action: "replace" } as const;
166 }),
167
168 read: Effect.fn(function* ({ id, output, olds }) {
169 const { accountId } = yield* yield* CloudflareEnvironment;
170 const acct = output?.accountId ?? accountId;
171 if (output?.pipelineId) {
172 const observed = yield* getPipeline(acct, output.pipelineId);
173 if (observed) return toAttributes(observed, acct);
174 }
175 // Cold read — pipeline names are unique per account; a
176 // generated-name match is proof of ownership, an explicit name is
177 // not.
178 const name = yield* pipelineName(id, olds?.name);
179 const match = yield* findPipelineByName(acct, name);
180 if (match) {
181 const attrs = toAttributes(match, acct);
182 return olds?.name !== undefined ? Unowned(attrs) : attrs;
183 }
184 return undefined;
185 }),
186
187 reconcile: Effect.fn(function* ({ id, news, output }) {
188 const { accountId } = yield* yield* CloudflareEnvironment;
189 const name = yield* pipelineName(id, news.name);
190 const sql = news.sql as string;
191
192 // 1. Observe — by cached id first, then by (unique) name so we
193 // recover from lost state writes.
194 let observed = output?.pipelineId
195 ? yield* getPipeline(output.accountId ?? accountId, output.pipelineId)
196 : undefined;
197 if (!observed) {
198 observed = yield* findPipelineByName(accountId, name);
199 }
200
201 // Converge drift — there is no update API, so when the observed SQL
202 // differs from the desired SQL (e.g. a referenced stream/sink was
203 // replaced and `sql` was unresolved at diff time), delete the stale
204 // pipeline and fall through to recreate it under the same name.
205 // Nothing references a pipeline downstream, so this is safe.
206 if (
207 observed &&
208 (canonicalSql(observed.sql) !== canonicalSql(sql) || observed.name !== name)
209 ) {
210 yield* deletePipeline(accountId, observed.id);
211 // Wait until the delete is visible so the recreate below does not
212 // race an `PipelineAlreadyExists` against the dying pipeline.
213 yield* getPipeline(accountId, observed.id).pipe(
214 Effect.repeat({
215 schedule: Schedule.max([Schedule.exponential("250 millis"), Schedule.recurs(8)]),
216 until: (p) => p === undefined,
217 }),
218 );
219 observed = undefined;
220 }
221
222 // 2. Ensure — create when missing. The referenced stream/sink may
223 // have been created moments ago, so ride out eventual-consistency
224 // `TableNotFound` blips with a bounded retry. An AlreadyExists is
225 // a race or recovery — resolve it via the name lookup. There is
226 // no sync step: the SQL is immutable, so changes arrive as
227 // replacements (diff).
228 if (!observed) {
229 observed = yield* pipelines.createV1Pipeline({ accountId, name, sql }).pipe(
230 Effect.retry({
231 while: (e) => e._tag === "TableNotFound",
232 schedule: Schedule.max([Schedule.exponential("500 millis"), Schedule.recurs(6)]),
233 }),
234 Effect.catchTag("PipelineAlreadyExists", (error) =>
235 findPipelineByName(accountId, name).pipe(
236 Effect.flatMap((match) => (match ? Effect.succeed(match) : Effect.fail(error))),
237 ),
238 ),
239 );
240 }
241
242 return toAttributes(observed, accountId);
243 }),
244
245 delete: Effect.fn(function* ({ output }) {
246 yield* deletePipeline(output.accountId, output.pipelineId);
247 }),
248
249 // Account collection — pipelines are account-scoped. Exhaustively
250 // paginate the account-wide list and hydrate each row into the same
251 // Attributes shape `read` returns.
252 list: Effect.fn(function* () {
253 const { accountId } = yield* yield* CloudflareEnvironment;
254 return yield* pipelines.listV1Pipeline.pages({ accountId }).pipe(
255 Stream.runCollect,
256 Effect.map((chunk) =>
257 Array.from(chunk).flatMap((page) =>
258 (page.result ?? []).map((p) => toAttributes(p, accountId)),
259 ),
260 ),
261 );
262 }),
263 });
264
265 /**
266 * Canonical SQL for change detection: whitespace runs outside quoted
267 * literals/identifiers collapse to one space, surrounding whitespace and
268 * a trailing `;` are dropped.
269 */
270 const canonicalSql = (sql: string): string => {
271 let out = "";
272 let quote: string | undefined;
273 let pendingSpace = false;
274 for (const ch of sql.trim().replace(/;\s*$/, "")) {
275 if (quote) {
276 out += ch;
277 if (ch === quote) quote = undefined;
278 } else if (/\s/.test(ch)) {
279 pendingSpace = true;
280 } else {
281 if (pendingSpace && out.length > 0) out += " ";
282 pendingSpace = false;
283 if (ch === "'" || ch === '"' || ch === "`") quote = ch;
284 out += ch;
285 }
286 }
287 return out;
288 };
289
290 /**
291 * The subset of pipeline state shared by get/list/create responses.
292 */
293 interface ObservedPipeline {
294 id: string;
295 name: string;
296 sql: string;
297 status: string;
298 createdAt: string;
299 modifiedAt: string;
300 }
301
302 // Pipelines entity names must be alphanumeric/underscore only, so swap
303 // the default hyphen delimiter for underscores.
304 const pipelineName = (id: string, name: string | undefined) =>
305 Effect.gen(function* () {
306 if (name) return name;
307 const generated = yield* createPhysicalName({
308 id,
309 lowercase: true,
310 delimiter: "_",
311 });
312 return generated.replaceAll(/[^a-zA-Z0-9_]/g, "_");
313 });
314
315 /**
316 * Read a pipeline by id, mapping "gone" (`PipelineNotExists`, Cloudflare
317 * error code 1000) to `undefined`.
318 */
319 const getPipeline = (accountId: string, pipelineId: string) =>
320 pipelines.getV1Pipeline({ accountId, pipelineId }).pipe(
321 Effect.map((p): ObservedPipeline | undefined => p),
322 Effect.catchTag("PipelineNotExists", () => Effect.undefined),
323 );
324
325 /**
326 * Idempotent delete — a pipeline that is already gone is success.
327 */
328 const deletePipeline = (accountId: string, pipelineId: string) =>
329 pipelines
330 .deleteV1Pipeline({ accountId, pipelineId })
331 .pipe(Effect.catchTag("PipelineNotExists", () => Effect.void));
332
333 const findPipelineByName = (accountId: string, name: string) =>
334 pipelines.listV1Pipeline.items({ accountId }).pipe(
335 Stream.runCollect,
336 Effect.map((chunk): ObservedPipeline | undefined =>
337 Array.from(chunk).find((p) => p.name === name),
338 ),
339 );
340
341 const toAttributes = (observed: ObservedPipeline, accountId: string): PipelineAttributes => ({
342 pipelineId: observed.id,
343 accountId,
344 name: observed.name,
345 sql: observed.sql,
346 status: observed.status,
347 createdAt: observed.createdAt,
348 modifiedAt: observed.modifiedAt,
349 });