@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 06:00 UTC

main
ts 400 lines 13.7 KB
Raw
1 import * as r2 from "@distilled.cloud/cloudflare/r2";
2 import { Unowned } from "alchemy/AdoptPolicy";
3 import { isResolved } from "alchemy/Diff";
4 import * as Provider from "alchemy/Provider";
5 import { Resource } from "alchemy/Resource";
6 import * as Effect from "effect/Effect";
7 import * as Predicate from "effect/Predicate";
8 import * as Redacted from "effect/Redacted";
9 import { CloudflareEnvironment } from "../CloudflareEnvironment.ts";
10 import type { Providers } from "../Providers.ts";
11 import type { Bucket } from "./Bucket.ts";
12
13 const TypeId = "Cloudflare.R2.BucketSippy" as const;
14 type TypeId = typeof TypeId;
15
16 /**
17 * Source bucket configuration for an AWS S3 origin.
18 */
19 export interface BucketSippyAwsSource {
20 /** Marks the source as an AWS S3 bucket. */
21 provider: "aws";
22 /**
23 * Name of the AWS S3 bucket to migrate objects from.
24 */
25 bucket: string;
26 /**
27 * AWS region the source bucket lives in, e.g. `us-east-1`.
28 */
29 region: string;
30 /**
31 * Access Key ID of an AWS IAM credential with read access to the
32 * source bucket. Write-only — Cloudflare never returns it.
33 */
34 accessKeyId: Redacted.Redacted<string>;
35 /**
36 * Secret Access Key paired with `accessKeyId`. Write-only.
37 */
38 secretAccessKey: Redacted.Redacted<string>;
39 }
40
41 /**
42 * Source bucket configuration for a Google Cloud Storage origin.
43 */
44 export interface BucketSippyGcsSource {
45 /** Marks the source as a Google Cloud Storage bucket. */
46 provider: "gcs";
47 /**
48 * Name of the GCS bucket to migrate objects from.
49 */
50 bucket: string;
51 /**
52 * Client email of a GCP service account with read access to the
53 * source bucket.
54 */
55 clientEmail: string;
56 /**
57 * Private key of the GCP service account. Write-only — Cloudflare
58 * never returns it.
59 */
60 privateKey: Redacted.Redacted<string>;
61 }
62
63 /**
64 * The upstream bucket Sippy pulls objects from on R2 cache misses.
65 */
66 export type BucketSippySource = BucketSippyAwsSource | BucketSippyGcsSource;
67
68 /**
69 * R2 credentials Sippy uses to write migrated objects into the
70 * destination bucket. Create an R2 API token with write access to the
71 * bucket and pass its S3-compatible credentials here.
72 */
73 export interface BucketSippyDestination {
74 /**
75 * Access Key ID of the R2 API token. Write-only at the secret level —
76 * Cloudflare echoes only the key ID back.
77 */
78 accessKeyId: Redacted.Redacted<string>;
79 /**
80 * Secret Access Key of the R2 API token. Write-only.
81 */
82 secretAccessKey: Redacted.Redacted<string>;
83 }
84
85 export interface BucketSippyProps {
86 /**
87 * Name of the R2 bucket to enable incremental migration into. Pass
88 * `bucket.bucketName` from a `Cloudflare.R2.Bucket`.
89 *
90 * Immutable — changing the bucket triggers a replacement.
91 */
92 bucketName: string;
93 /**
94 * Jurisdiction of the bucket (must match the bucket's own
95 * jurisdiction).
96 *
97 * Immutable — changing the jurisdiction triggers a replacement.
98 * @default "default"
99 */
100 jurisdiction?: Bucket.Jurisdiction;
101 /**
102 * The upstream AWS S3 or Google Cloud Storage bucket to migrate
103 * objects from. Source credentials are write-only — they are sent to
104 * Cloudflare but never read back.
105 */
106 source: BucketSippySource;
107 /**
108 * R2 API token credentials Sippy uses to write objects into the
109 * destination bucket.
110 */
111 destination: BucketSippyDestination;
112 }
113
114 export interface BucketSippyAttributes {
115 /** Name of the R2 bucket Sippy is enabled on. */
116 bucketName: string;
117 /** Account the bucket lives in. */
118 accountId: string;
119 /** Jurisdiction of the bucket. */
120 jurisdiction: Bucket.Jurisdiction;
121 /** Whether Sippy is currently enabled on the bucket. */
122 enabled: boolean;
123 /** The configured source bucket as reported by Cloudflare (sans secrets). */
124 source: BucketSippy.SourceAttributes;
125 /** The configured destination as reported by Cloudflare (sans secrets). */
126 destination: BucketSippy.DestinationAttributes;
127 }
128
129 export type BucketSippy = Resource<
130 TypeId,
131 BucketSippyProps,
132 BucketSippyAttributes,
133 never,
134 Providers
135 >;
136
137 /**
138 * Sippy — incremental migration from AWS S3 or Google Cloud Storage
139 * into a Cloudflare R2 bucket.
140 *
141 * When Sippy is enabled on a bucket, any object requested from R2 that
142 * is not yet present is fetched from the configured source bucket,
143 * served, and copied into R2 — migrating data on demand without a bulk
144 * transfer and without paying double storage during the transition.
145 *
146 * One Sippy configuration exists per bucket (it is a singleton
147 * sub-resource of the bucket). Destroying the resource disables Sippy;
148 * objects already migrated stay in the R2 bucket.
149 * ### Migrating from AWS S3
150 * **Example:** Enable Sippy on a bucket with an S3 source
151 * ```typescript
152 * const bucket = yield* Cloudflare.R2.Bucket("Media");
153 *
154 * yield* Cloudflare.R2.BucketSippy("MediaMigration", {
155 * bucketName: bucket.bucketName,
156 * source: {
157 * provider: "aws",
158 * bucket: "legacy-media",
159 * region: "us-east-1",
160 * accessKeyId: yield* Config.Redacted("AWS_ACCESS_KEY_ID"),
161 * secretAccessKey: yield* Config.Redacted("AWS_SECRET_ACCESS_KEY"),
162 * },
163 * destination: {
164 * accessKeyId: yield* Config.Redacted("R2_ACCESS_KEY_ID"),
165 * secretAccessKey: yield* Config.Redacted("R2_SECRET_ACCESS_KEY"),
166 * },
167 * });
168 * ```
169 *
170 * ### Migrating from Google Cloud Storage
171 * **Example:** Enable Sippy with a GCS source
172 * ```typescript
173 * yield* Cloudflare.R2.BucketSippy("MediaMigration", {
174 * bucketName: bucket.bucketName,
175 * source: {
176 * provider: "gcs",
177 * bucket: "legacy-media",
178 * clientEmail: "sippy@my-project.iam.gserviceaccount.com",
179 * privateKey: yield* Config.Redacted("GCS_PRIVATE_KEY"),
180 * },
181 * destination: {
182 * accessKeyId: yield* Config.Redacted("R2_ACCESS_KEY_ID"),
183 * secretAccessKey: yield* Config.Redacted("R2_SECRET_ACCESS_KEY"),
184 * },
185 * });
186 * ```
187 *
188 * @see https://developers.cloudflare.com/r2/data-migration/sippy/
189 *
190 * @resource
191 * @product R2
192 * @category Storage & Databases
193 */
194 export const BucketSippy = Resource<BucketSippy>(TypeId);
195
196 export declare namespace BucketSippy {
197 /**
198 * The source bucket as echoed back by Cloudflare. Secrets are never
199 * returned.
200 */
201 export type SourceAttributes = {
202 provider: "aws" | "gcs" | undefined;
203 bucket: string | undefined;
204 region: string | undefined;
205 bucketUrl: string | undefined;
206 };
207 /**
208 * The destination as echoed back by Cloudflare. Only the access key
209 * ID is returned, never the secret.
210 */
211 export type DestinationAttributes = {
212 provider: "r2" | (string & {}) | undefined;
213 account: string | undefined;
214 bucket: string | undefined;
215 accessKeyId: string | undefined;
216 };
217 }
218
219 /**
220 * Returns true if the given value is an BucketSippy resource.
221 */
222 export const isBucketSippy = (value: unknown): value is BucketSippy =>
223 Predicate.hasProperty(value, "Type") && value.Type === TypeId;
224
225 export const BucketSippyProvider = () =>
226 Provider.succeed(BucketSippy, {
227 stables: ["bucketName", "accountId", "jurisdiction"],
228
229 diff: Effect.fn(function* ({ olds, news }) {
230 if (!isResolved(news)) return undefined;
231 // The bucket is the configuration's identity — moving Sippy to a
232 // different bucket (or jurisdiction) is a replacement. Compare
233 // only once both sides are concrete strings.
234 if (
235 typeof olds.bucketName === "string" &&
236 typeof news.bucketName === "string" &&
237 olds.bucketName !== news.bucketName
238 ) {
239 return { action: "replace" } as const;
240 }
241 if (
242 olds.bucketName !== undefined &&
243 (olds.jurisdiction ?? "default") !== (news.jurisdiction ?? "default")
244 ) {
245 return { action: "replace" } as const;
246 }
247 return undefined;
248 }),
249
250 read: Effect.fn(function* ({ output, olds }) {
251 const { accountId } = yield* yield* CloudflareEnvironment;
252 const acct = output?.accountId ?? accountId;
253 // The bucket is the configuration's identity; cold reads derive it
254 // from the last-persisted props.
255 const bucketName =
256 output?.bucketName ?? (typeof olds?.bucketName === "string" ? olds.bucketName : undefined);
257 if (bucketName === undefined) return undefined;
258 const jurisdiction = output?.jurisdiction ?? olds?.jurisdiction ?? "default";
259
260 const observed = yield* r2.getBucketSippy({ accountId: acct, bucketName, jurisdiction }).pipe(
261 Effect.map((config): r2.GetBucketSippyResponse | undefined => config),
262 // The bucket itself is gone — so is its Sippy configuration.
263 Effect.catchTag("NoSuchBucket", () => Effect.undefined),
264 );
265 // A bucket with Sippy never configured (or disabled) reads back as
266 // `{ enabled: false }` — that is "absent" for this resource.
267 if (!observed || observed.enabled !== true) return undefined;
268
269 const attrs = toAttributes(observed, acct, bucketName, jurisdiction);
270 // Sippy configs carry no ownership markers. With no prior output we
271 // cannot prove we enabled it — brand it `Unowned` so takeover is
272 // gated behind the adopt policy.
273 return output ? attrs : Unowned(attrs);
274 }),
275
276 reconcile: Effect.fn(function* ({ news, output }) {
277 const { accountId } = yield* yield* CloudflareEnvironment;
278 const acct = output?.accountId ?? accountId;
279 // Inputs have been resolved to concrete strings by Plan.
280 const bucketName = news.bucketName as string;
281 const jurisdiction = news.jurisdiction ?? "default";
282
283 // The PUT is a full upsert of the bucket's single Sippy
284 // configuration — enabling when absent and re-configuring when
285 // present — so observe/ensure/sync collapse into one idempotent
286 // call. Source/destination secrets are write-only, so an observed
287 // config can never be diffed against the desired one anyway.
288 const synced = yield* r2.putBucketSippy({
289 accountId: acct,
290 bucketName,
291 jurisdiction,
292 source: toRequestSource(news.source),
293 destination: {
294 provider: "r2",
295 accessKeyId: Redacted.value(news.destination.accessKeyId),
296 secretAccessKey: Redacted.value(news.destination.secretAccessKey),
297 },
298 });
299
300 return toAttributes(synced, acct, bucketName, jurisdiction);
301 }),
302
303 delete: Effect.fn(function* ({ output }) {
304 // Disabling Sippy on a bucket where it is already disabled is a
305 // success (`{ enabled: false }`); a missing bucket means there is
306 // nothing left to disable — both make delete idempotent.
307 yield* r2
308 .deleteBucketSippy({
309 accountId: output.accountId,
310 bucketName: output.bucketName,
311 jurisdiction: output.jurisdiction,
312 })
313 .pipe(Effect.catchTag("NoSuchBucket", () => Effect.void));
314 }),
315
316 // Sippy is a per-bucket singleton with no account-wide enumeration
317 // API, so fan out from the parent: enumerate every R2 bucket, read its
318 // Sippy config with bounded concurrency, and emit one item per bucket
319 // that actually has Sippy enabled (mirroring `read`, which treats a
320 // disabled/absent config as "not present").
321 list: Effect.fn(function* () {
322 const { accountId } = yield* yield* CloudflareEnvironment;
323 const { buckets } = yield* r2.listBuckets({ accountId });
324 const perBucket = yield* Effect.forEach(
325 buckets ?? [],
326 (bucket) => {
327 const bucketName = bucket.name;
328 if (bucketName == null) {
329 return Effect.succeed([] as BucketSippyAttributes[]);
330 }
331 const jurisdiction = (bucket.jurisdiction ?? "default") as Bucket.Jurisdiction;
332 return r2.getBucketSippy({ accountId, bucketName, jurisdiction }).pipe(
333 Effect.map((config) =>
334 config.enabled === true
335 ? [toAttributes(config, accountId, bucketName, jurisdiction)]
336 : [],
337 ),
338 // A bucket that vanished mid-enumeration, or one whose plan
339 // rejects the Sippy route, contributes nothing — skip it.
340 Effect.catchTag(["NoSuchBucket", "InvalidRoute", "Forbidden"], () =>
341 Effect.succeed([] as BucketSippyAttributes[]),
342 ),
343 );
344 },
345 { concurrency: 10 },
346 );
347 return perBucket.flat();
348 }),
349 });
350
351 /**
352 * Build the request body shape the distilled `putBucketSippy` method
353 * accepts. Secrets are unwrapped here because the distilled TS types
354 * declare the credential fields as plain strings.
355 */
356 const toRequestSource = (source: BucketSippySource) =>
357 source.provider === "aws"
358 ? {
359 provider: "aws" as const,
360 bucket: source.bucket,
361 region: source.region,
362 accessKeyId: Redacted.value(source.accessKeyId),
363 secretAccessKey: Redacted.value(source.secretAccessKey),
364 }
365 : {
366 provider: "gcs" as const,
367 bucket: source.bucket,
368 clientEmail: source.clientEmail,
369 privateKey: Redacted.value(source.privateKey),
370 };
371
372 const toAttributes = (
373 config: r2.GetBucketSippyResponse,
374 accountId: string,
375 bucketName: string,
376 jurisdiction: Bucket.Jurisdiction,
377 ): BucketSippyAttributes => ({
378 bucketName,
379 accountId,
380 jurisdiction,
381 enabled: config.enabled ?? false,
382 source: {
383 // Cloudflare reports AWS sources as either "aws" or the legacy "s3".
384 provider:
385 config.source?.provider === "gcs"
386 ? "gcs"
387 : config.source?.provider != null
388 ? "aws"
389 : undefined,
390 bucket: config.source?.bucket ?? undefined,
391 region: config.source?.region ?? undefined,
392 bucketUrl: config.source?.bucketUrl ?? undefined,
393 },
394 destination: {
395 provider: config.destination?.provider ?? undefined,
396 account: config.destination?.account ?? undefined,
397 bucket: config.destination?.bucket ?? undefined,
398 accessKeyId: config.destination?.accessKeyId ?? undefined,
399 },
400 });