@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 511 lines 17 KB
Raw
1 import { Credentials } from "@distilled.cloud/cloudflare/Credentials";
2 import * as pipelines from "@distilled.cloud/cloudflare/pipelines";
3 import { Unowned } from "alchemy/AdoptPolicy";
4 import { isResolved } from "alchemy/Diff";
5 import { createPhysicalName } from "alchemy/PhysicalName";
6 import * as Provider from "alchemy/Provider";
7 import { Resource } from "alchemy/Resource";
8 import * as Effect from "effect/Effect";
9 import type * as HttpClient from "effect/http/HttpClient";
10 import * as Predicate from "effect/Predicate";
11 import * as Redacted from "effect/Redacted";
12 import { CloudflareEnvironment } from "../CloudflareEnvironment.ts";
13 import type { Providers } from "../Providers.ts";
14
15 const TypeId = "Cloudflare.Pipelines.LegacyPipeline" as const;
16 type TypeId = typeof TypeId;
17
18 /**
19 * HTTP ingest source — events are POSTed as JSON to the pipeline's
20 * `endpoint` URL.
21 */
22 export interface LegacyPipelineHttpSource {
23 /** Accept events over HTTP at the pipeline endpoint. */
24 type: "http";
25 /**
26 * Require Cloudflare API-token authentication on the ingest endpoint.
27 * @default false
28 */
29 authentication?: boolean;
30 /**
31 * CORS configuration for browser-originated ingestion.
32 */
33 cors?: {
34 /** Allowed origins, e.g. `["https://example.com"]` or `["*"]`. */
35 origins?: string[];
36 };
37 }
38
39 /**
40 * Worker binding ingest source — events are sent from a Worker via a
41 * `pipelines` binding.
42 */
43 export interface LegacyPipelineBindingSource {
44 /** Accept events from a Worker `pipelines` binding. */
45 type: "binding";
46 }
47
48 /**
49 * An ingest source of a legacy pipeline.
50 */
51 export type LegacyPipelineSource = LegacyPipelineHttpSource | LegacyPipelineBindingSource;
52
53 /**
54 * R2 destination configuration of a legacy pipeline.
55 */
56 export interface LegacyPipelineDestination {
57 /**
58 * Name of the destination R2 bucket. The bucket must already exist.
59 */
60 bucket: string;
61 /**
62 * R2 S3-compatible credentials the pipeline uses to write objects.
63 * Write-only — Cloudflare never echoes them back.
64 */
65 credentials: {
66 /** R2 access key id (the API token id). */
67 accessKeyId: Redacted.Redacted<string>;
68 /** R2 secret access key (SHA-256 hex of the API token value). */
69 secretAccessKey: Redacted.Redacted<string>;
70 /**
71 * S3-compatible endpoint of the R2 bucket.
72 * @default https://{accountId}.r2.cloudflarestorage.com
73 */
74 endpoint?: string;
75 };
76 /**
77 * Batching policy controlling when the pipeline flushes a batch of
78 * events to R2.
79 */
80 batch?: {
81 /** Flush once the batch reaches this size in bytes. */
82 maxBytes?: number;
83 /** Flush at most this many seconds after the batch was opened. */
84 maxDurationS?: number;
85 /** Flush once the batch reaches this many events. */
86 maxRows?: number;
87 };
88 /**
89 * Compression applied to output files.
90 * @default "gzip"
91 */
92 compression?: "none" | "gzip" | "deflate";
93 /**
94 * Key prefix under which output objects are written.
95 */
96 prefix?: string;
97 /**
98 * Time-partitioned directory layout of output object keys
99 * (strftime-style, e.g. `event_date=%F/hr=%H`).
100 */
101 filepath?: string;
102 /**
103 * Name template of output files within a partition.
104 */
105 filename?: string;
106 }
107
108 export interface LegacyPipelineProps {
109 /**
110 * Name of the pipeline. Unique per account; the legacy API addresses
111 * pipelines by name, so changing it triggers a replacement. If
112 * omitted, a unique name is generated from the app, stage, and
113 * logical ID.
114 * @default ${app}-${id}-${stage}-${suffix}
115 */
116 name?: string;
117 /**
118 * Ingest sources accepted by the pipeline.
119 * @default [{ type: "http" }, { type: "binding" }]
120 */
121 source?: LegacyPipelineSource[];
122 /**
123 * R2 destination the pipeline batches events into.
124 */
125 destination: LegacyPipelineDestination;
126 }
127
128 export interface LegacyPipelineAttributes {
129 /** Cloudflare-assigned pipeline identifier. */
130 pipelineId: string;
131 /** Account that owns the pipeline. */
132 accountId: string;
133 /** Pipeline name (unique per account; the API identifier). */
134 name: string;
135 /** HTTP endpoint URL events can be POSTed to. */
136 endpoint: string;
137 /** Destination R2 bucket name. */
138 bucket: string;
139 /** Version number of the last saved configuration. */
140 version: number;
141 }
142
143 export type LegacyPipeline = Resource<
144 TypeId,
145 LegacyPipelineProps,
146 LegacyPipelineAttributes,
147 never,
148 Providers
149 >;
150
151 /**
152 * A **legacy** Cloudflare Pipeline — the original HTTP-ingest → R2 batch
153 * product (`/accounts/{account}/pipelines`).
154 *
155 * :::caution
156 * This is the **deprecated, legacy** Pipelines API. Cloudflare has
157 * superseded it with the SQL-based product — prefer
158 * {@link Stream}, {@link Sink}, and {@link Pipeline}
159 * for new infrastructure. This resource exists only to manage
160 * pre-existing legacy pipelines.
161 * :::
162 *
163 * A legacy pipeline accepts JSON events over HTTP (and/or a Worker
164 * `pipelines` binding) and batches them into an R2 bucket using
165 * S3-compatible credentials.
166 * ### Creating a Legacy Pipeline
167 * **Example:** HTTP ingest into R2
168 * The S3-compatible credentials are derived from a Cloudflare API token:
169 * the access key id is the token id and the secret is the SHA-256 hex
170 * digest of the token value.
171 * ```typescript
172 * const bucket = yield* Cloudflare.R2.Bucket("events", {});
173 *
174 * const pipeline = yield* Cloudflare.Pipelines.LegacyPipeline("ingest", {
175 * destination: {
176 * bucket: bucket.bucketName,
177 * credentials: {
178 * accessKeyId: yield* Config.Redacted("R2_ACCESS_KEY_ID"),
179 * secretAccessKey: yield* Config.Redacted("R2_SECRET_ACCESS_KEY"),
180 * },
181 * },
182 * });
183 * // POST events to pipeline.endpoint
184 * ```
185 *
186 * **Example:** Tuned batching and CORS
187 * ```typescript
188 * const pipeline = yield* Cloudflare.Pipelines.LegacyPipeline("ingest", {
189 * source: [
190 * { type: "http", cors: { origins: ["https://example.com"] } },
191 * ],
192 * destination: {
193 * bucket: bucket.bucketName,
194 * credentials,
195 * batch: { maxDurationS: 10, maxRows: 1000 },
196 * compression: "gzip",
197 * prefix: "ingest",
198 * },
199 * });
200 * ```
201 *
202 * @see https://developers.cloudflare.com/pipelines/
203 *
204 * @resource
205 * @product Pipelines
206 * @category Storage & Databases
207 */
208 export const LegacyPipeline = Resource<LegacyPipeline>(TypeId);
209
210 /**
211 * Returns true if the given value is a LegacyPipeline resource.
212 */
213 export const isLegacyPipeline = (value: unknown): value is LegacyPipeline =>
214 Predicate.hasProperty(value, "Type") && value.Type === TypeId;
215
216 export const LegacyPipelineProvider = () =>
217 Provider.succeed(LegacyPipeline, {
218 stables: ["pipelineId", "accountId", "name", "endpoint"],
219
220 diff: Effect.fn(function* ({ id, olds, news, output }) {
221 const { accountId } = yield* yield* CloudflareEnvironment;
222 if (!isResolved(news)) return undefined;
223 if ((output?.accountId ?? accountId) !== accountId) {
224 return { action: "replace" } as const;
225 }
226 const o = olds as LegacyPipelineProps | undefined;
227 if (o === undefined) return undefined;
228 // The legacy API addresses pipelines by name — a name change is a
229 // replacement. Everything else updates in place via PUT.
230 const newName = yield* legacyPipelineName(id, news.name);
231 const oldName = output?.name ?? (yield* legacyPipelineName(id, o.name));
232 if (newName !== oldName) {
233 return { action: "replace" } as const;
234 }
235 return undefined;
236 }),
237
238 list: Effect.fn(function* () {
239 const { accountId } = yield* yield* CloudflareEnvironment;
240 // The list endpoint returns summary items (id/name/endpoint only),
241 // so hydrate each by name into the exact `read` Attributes shape
242 // with bounded concurrency and a typed per-item not-found skip
243 // (a pipeline can vanish between the list and the get).
244 const summaries = yield* listLegacyPipelineSummaries(accountId);
245 const rows = yield* Effect.forEach(
246 summaries,
247 (summary) =>
248 getLegacyPipeline(accountId, summary.name).pipe(
249 Effect.map((observed) => (observed ? toAttributes(observed, accountId) : undefined)),
250 ),
251 { concurrency: 10 },
252 );
253 return rows.filter((row): row is LegacyPipelineAttributes => row !== undefined);
254 }),
255
256 read: Effect.fn(function* ({ id, output, olds }) {
257 const { accountId } = yield* yield* CloudflareEnvironment;
258 const acct = output?.accountId ?? accountId;
259 if (output?.name) {
260 const observed = yield* getLegacyPipeline(acct, output.name);
261 if (observed) return toAttributes(observed, acct);
262 }
263 // Cold read — names are unique per account; a generated-name match
264 // is proof of ownership, an explicit name is not.
265 const name = yield* legacyPipelineName(id, olds?.name);
266 const observed = yield* getLegacyPipeline(acct, name);
267 if (observed) {
268 const attrs = toAttributes(observed, acct);
269 return olds?.name !== undefined ? Unowned(attrs) : attrs;
270 }
271 return undefined;
272 }),
273
274 reconcile: Effect.fn(function* ({ id, news, olds, output }) {
275 const { accountId } = yield* yield* CloudflareEnvironment;
276 const name = yield* legacyPipelineName(id, news.name);
277
278 // 1. Observe — the name is the API identifier, so a single get
279 // covers both the cached-output and cold-recovery cases.
280 let observed = yield* getLegacyPipeline(output?.accountId ?? accountId, output?.name ?? name);
281
282 // 2. Ensure — create when missing.
283 if (!observed) {
284 observed = yield* pipelines.createPipeline({
285 accountId,
286 name,
287 source: toRequestSource(news.source),
288 destination: toRequestDestination(accountId, news.destination),
289 });
290 return toAttributes(observed, accountId);
291 }
292
293 // 3. Sync — diff observed config against desired and PUT the full
294 // desired state only when something changed. Credentials are
295 // write-only, so `olds` is the only (best-effort) baseline for
296 // them: on adoption (`olds` undefined) or a credential change we
297 // push an update unconditionally.
298 const credsRotated =
299 olds === undefined ||
300 Redacted.value(news.destination.credentials.accessKeyId) !==
301 Redacted.value(olds.destination.credentials.accessKeyId) ||
302 Redacted.value(news.destination.credentials.secretAccessKey) !==
303 Redacted.value(olds.destination.credentials.secretAccessKey);
304 if (credsRotated || drifted(observed, news)) {
305 observed = yield* pipelines.updatePipeline({
306 accountId,
307 pipelineName: observed.name,
308 name,
309 source: toRequestSource(news.source),
310 destination: toRequestDestination(accountId, news.destination),
311 });
312 }
313
314 return toAttributes(observed, accountId);
315 }),
316
317 delete: Effect.fn(function* ({ output }) {
318 yield* pipelines
319 .deletePipeline({
320 accountId: output.accountId,
321 pipelineName: output.name,
322 })
323 .pipe(Effect.catchTag("PipelineNotExists", () => Effect.void));
324 }),
325 });
326
327 /**
328 * The pipeline state shared by get/create/update responses.
329 */
330 interface ObservedLegacyPipeline {
331 id: string;
332 name: string;
333 endpoint: string;
334 version: number;
335 destination: {
336 batch: { maxBytes: number; maxDurationS: number; maxRows: number };
337 compression: { type: string };
338 path: {
339 bucket: string;
340 filename?: string | null;
341 filepath?: string | null;
342 prefix?: string | null;
343 };
344 };
345 source: readonly {
346 type: string;
347 authentication?: boolean | null;
348 cors?: { origins?: readonly string[] | null } | null;
349 }[];
350 }
351
352 const legacyPipelineName = (id: string, name: string | undefined) =>
353 Effect.gen(function* () {
354 if (name) return name;
355 return yield* createPhysicalName({ id, lowercase: true, delimiter: "-" });
356 });
357
358 /**
359 * Read a legacy pipeline by name, mapping "gone" (`PipelineNotExists`,
360 * Cloudflare error code 1000) to `undefined`.
361 */
362 const getLegacyPipeline = (accountId: string, pipelineName: string) =>
363 pipelines.getPipeline({ accountId, pipelineName }).pipe(
364 Effect.map((p): ObservedLegacyPipeline | undefined => p),
365 Effect.catchTag("PipelineNotExists", () => Effect.undefined),
366 );
367
368 interface LegacyPipelineSummary {
369 id: string;
370 name: string;
371 endpoint: string;
372 }
373
374 /**
375 * Enumerate every legacy pipeline summary in an account. The distilled
376 * `listPipelines` op is not stream-paginated, so walk the `page`/
377 * `per_page` query params using `result_info.total_count` (falling back
378 * to a short-page sentinel) until every page is collected. The list
379 * endpoint caps `per_page` at 50 and returns only summary fields.
380 */
381 const listLegacyPipelineSummaries = (accountId: string) => {
382 const perPage = 50;
383 const collect = (
384 page: number,
385 acc: LegacyPipelineSummary[],
386 ): Effect.Effect<
387 LegacyPipelineSummary[],
388 pipelines.ListPipelinesError,
389 Credentials | HttpClient.HttpClient
390 > =>
391 Effect.gen(function* () {
392 const response = yield* pipelines.listPipelines({
393 accountId,
394 page: String(page),
395 perPage: String(perPage),
396 });
397 const results = (response.results ?? []).map((p): LegacyPipelineSummary => ({
398 id: p.id,
399 name: p.name ?? "",
400 endpoint: p.endpoint ?? "",
401 }));
402 const next = [...acc, ...results];
403 const total = response.resultInfo?.totalCount;
404 const done = results.length < perPage || (total != null && next.length >= total);
405 return done ? next : yield* collect(page + 1, next);
406 });
407 return collect(1, []);
408 };
409
410 const defaultSources: LegacyPipelineSource[] = [{ type: "http" }, { type: "binding" }];
411
412 const toRequestSource = (source: LegacyPipelineSource[] | undefined) =>
413 (source ?? defaultSources).map((s) =>
414 s.type === "http"
415 ? {
416 format: "json" as const,
417 type: s.type,
418 authentication: s.authentication,
419 cors: s.cors,
420 }
421 : { format: "json" as const, type: s.type },
422 );
423
424 const toRequestDestination = (accountId: string, destination: LegacyPipelineDestination) => ({
425 type: "r2" as const,
426 format: "json" as const,
427 batch: destination.batch ?? {},
428 compression: { type: destination.compression },
429 credentials: {
430 accessKeyId: Redacted.value(destination.credentials.accessKeyId),
431 secretAccessKey: Redacted.value(destination.credentials.secretAccessKey),
432 endpoint: destination.credentials.endpoint ?? `https://${accountId}.r2.cloudflarestorage.com`,
433 },
434 path: {
435 bucket: destination.bucket as string,
436 prefix: destination.prefix,
437 filepath: destination.filepath,
438 filename: destination.filename,
439 },
440 });
441
442 /**
443 * Detect drift between echoed cloud config and desired props. Only
444 * user-declared optional fields are diffed so we don't fight server-side
445 * defaults; credentials are write-only and handled separately.
446 */
447 const drifted = (observed: ObservedLegacyPipeline, news: LegacyPipelineProps): boolean => {
448 const d = news.destination;
449 const path = observed.destination.path;
450 if (path.bucket !== (d.bucket as string)) return true;
451 if (d.prefix !== undefined && d.prefix !== (path.prefix ?? undefined)) {
452 return true;
453 }
454 if (d.filepath !== undefined && d.filepath !== (path.filepath ?? undefined)) {
455 return true;
456 }
457 if (d.filename !== undefined && d.filename !== (path.filename ?? undefined)) {
458 return true;
459 }
460 if (d.compression !== undefined && d.compression !== observed.destination.compression.type) {
461 return true;
462 }
463 const batch = observed.destination.batch;
464 if (d.batch?.maxBytes !== undefined && d.batch.maxBytes !== batch.maxBytes) {
465 return true;
466 }
467 if (d.batch?.maxDurationS !== undefined && d.batch.maxDurationS !== batch.maxDurationS) {
468 return true;
469 }
470 if (d.batch?.maxRows !== undefined && d.batch.maxRows !== batch.maxRows) {
471 return true;
472 }
473 // Sources — compare the declared set of source types and the declared
474 // http options against what Cloudflare echoes.
475 const desired = news.source ?? defaultSources;
476 const observedTypes = observed.source.map((s) => s.type).sort();
477 const desiredTypes = desired.map((s) => s.type).sort();
478 if (observedTypes.join(",") !== desiredTypes.join(",")) return true;
479 const desiredHttp = desired.find((s): s is LegacyPipelineHttpSource => s.type === "http");
480 const observedHttp = observed.source.find((s) => s.type === "http");
481 if (desiredHttp && observedHttp) {
482 if (
483 desiredHttp.authentication !== undefined &&
484 desiredHttp.authentication !== (observedHttp.authentication ?? false)
485 ) {
486 return true;
487 }
488 if (desiredHttp.cors?.origins !== undefined) {
489 const observedOrigins = observedHttp.cors?.origins ?? [];
490 if (
491 desiredHttp.cors.origins.length !== observedOrigins.length ||
492 desiredHttp.cors.origins.some((o, i) => o !== observedOrigins[i])
493 ) {
494 return true;
495 }
496 }
497 }
498 return false;
499 };
500
501 const toAttributes = (
502 observed: ObservedLegacyPipeline,
503 accountId: string,
504 ): LegacyPipelineAttributes => ({
505 pipelineId: observed.id,
506 accountId,
507 name: observed.name,
508 endpoint: observed.endpoint,
509 bucket: observed.destination.path.bucket,
510 version: observed.version,
511 });